From noreply at buildbot.pypy.org Sat Aug 1 12:06:54 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 12:06:54 +0200 (CEST) Subject: [pypy-commit] pypy default: Workaround for issue #2102. Message-ID: <20150801100654.CE1521C1278@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78733:f63efb5d2ffa Date: 2015-08-01 12:07 +0200 http://bitbucket.org/pypy/pypy/changeset/f63efb5d2ffa/ Log: Workaround for issue #2102. Can't write a small test for this case, and I've no clue if in this case it means that invalid codemap was produced earlier. diff --git a/rpython/jit/backend/llsupport/codemap.py b/rpython/jit/backend/llsupport/codemap.py --- a/rpython/jit/backend/llsupport/codemap.py +++ b/rpython/jit/backend/llsupport/codemap.py @@ -157,6 +157,8 @@ self.l.append(0) # second marker def leave_portal_frame(self, jd_id, relpos): + if len(self.patch_position) < 1: + return # XXX should not occur, but does (issue #2102) to_patch = self.patch_position.pop() self.l[to_patch] = relpos self.l[to_patch + 1] = len(self.l) From noreply at buildbot.pypy.org Sat Aug 1 12:37:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 12:37:11 +0200 (CEST) Subject: [pypy-commit] pypy default: In case this mmap() fails with a different error code than ENOMEM, Message-ID: <20150801103711.124CB1C06D2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78734:f7c3c4ea78ef Date: 2015-08-01 12:37 +0200 http://bitbucket.org/pypy/pypy/changeset/f7c3c4ea78ef/ Log: In case this mmap() fails with a different error code than ENOMEM, complain explicitly. diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -719,6 +719,9 @@ so the memory has the executable bit set and gets allocated internally in case of a sandboxed process. """ + from errno import ENOMEM + from rpython.rlib import debug + if _CYGWIN: # XXX: JIT memory should be using mmap MAP_PRIVATE with # PROT_EXEC but Cygwin's fork() fails. mprotect() @@ -733,6 +736,14 @@ # are passed a non-zero address. Try again. res = alloc_hinted(rffi.cast(PTR, 0), map_size) if res == rffi.cast(PTR, -1): + # ENOMEM simply raises MemoryError, but other errors are fatal + if rposix.get_saved_errno() != ENOMEM: + debug.fatalerror_notb( + "Got an unexpected error trying to allocate some " + "memory for the JIT (tried to do mmap() with " + "PROT_EXEC|PROT_READ|PROT_WRITE). This can be caused " + "by a system policy like PAX. You need to find how " + "to work around the policy on your system.") raise MemoryError else: hint.pos += map_size From noreply at buildbot.pypy.org Sat Aug 1 14:55:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 14:55:21 +0200 (CEST) Subject: [pypy-commit] pypy default: Add another sanity test: rarithmetic.maxint == sys.maxint for now Message-ID: <20150801125521.0C3041C1247@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78735:3fe55025bf42 Date: 2015-08-01 14:55 +0200 http://bitbucket.org/pypy/pypy/changeset/3fe55025bf42/ Log: Add another sanity test: rarithmetic.maxint == sys.maxint for now diff --git a/rpython/rlib/rarithmetic.py b/rpython/rlib/rarithmetic.py --- a/rpython/rlib/rarithmetic.py +++ b/rpython/rlib/rarithmetic.py @@ -155,6 +155,8 @@ # the replacement for sys.maxint maxint = int(LONG_TEST - 1) +# for now, it should be equal to sys.maxint on all supported platforms +assert maxint == sys.maxint def is_valid_int(r): if objectmodel.we_are_translated(): From noreply at buildbot.pypy.org Sat Aug 1 16:06:41 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 16:06:41 +0200 (CEST) Subject: [pypy-commit] cffi default: From https://github.com/pyca/cryptography/pull/2156: add the pre_run() Message-ID: <20150801140641.A64431C03F4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r2242:b989307a75ac Date: 2015-08-01 16:07 +0200 http://bitbucket.org/cffi/cffi/changeset/b989307a75ac/ Log: From https://github.com/pyca/cryptography/pull/2156: add the pre_run() hook that is called just before writing C code. Limited to setuptools-based setup.py files. diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py --- a/cffi/setuptools_ext.py +++ b/cffi/setuptools_ext.py @@ -81,10 +81,16 @@ allsources.extend(kwds.pop('sources', [])) ext = Extension(name=module_name, sources=allsources, **kwds) - def make_mod(tmpdir): + def make_mod(tmpdir, pre_run=None): c_file = os.path.join(tmpdir, module_name + source_extension) log.info("generating cffi module %r" % c_file) mkpath(tmpdir) + # a setuptools-only, API-only hook: called with the "ext" and "ffi" + # arguments just before we turn the ffi into C code. To use it, + # subclass the 'distutils.command.build_ext.build_ext' class and + # add a method 'def pre_run(self, ext, ffi)'. + if pre_run is not None: + pre_run(ext, ffi) updated = recompiler.make_c_source(ffi, module_name, source, c_file) if not updated: log.info("already up-to-date") @@ -98,7 +104,8 @@ class build_ext_make_mod(base_class): def run(self): if ext.sources[0] == '$PLACEHOLDER': - ext.sources[0] = make_mod(self.build_temp) + pre_run = getattr(self, 'pre_run', None) + ext.sources[0] = make_mod(self.build_temp, pre_run) base_class.run(self) dist.cmdclass['build_ext'] = build_ext_make_mod # NB. multiple runs here will create multiple 'build_ext_make_mod' diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py --- a/testing/cffi1/test_zdist.py +++ b/testing/cffi1/test_zdist.py @@ -317,15 +317,28 @@ import cffi ffi = cffi.FFI() ffi.set_source("pack3.mymod", "/*code would be here*/") + ffi._hi_there = 42 """) with open("setup.py", "w") as f: f.write("""if 1: from setuptools import setup + from distutils.command.build_ext import build_ext + import os + + class TestBuildExt(build_ext): + def pre_run(self, ext, ffi): + assert ffi._hi_there == 42 + assert ext.name == "pack3.mymod" + fn = os.path.join(self.build_temp, '..', '..', 'see_me') + open(fn, 'w').close() + setup(name='example1', version='0.1', packages=['pack3'], package_dir={'': 'src1'}, - cffi_modules=["src1/pack3/_build.py:ffi"]) + cffi_modules=["src1/pack3/_build.py:ffi"], + cmdclass={'build_ext': TestBuildExt}, + ) """) @chdir_to_tmp @@ -334,6 +347,7 @@ self.run(["setup.py", "build"]) self.check_produced_files({'setup.py': None, 'build': '?', + 'see_me': None, 'src1': {'pack3': {'__init__.py': None, '_build.py': None}}}) @@ -343,6 +357,7 @@ self.run(["setup.py", "build_ext", "-i"]) self.check_produced_files({'setup.py': None, 'build': '?', + 'see_me': None, 'src1': {'pack3': {'__init__.py': None, '_build.py': None, 'mymod.SO': None}}}) From noreply at buildbot.pypy.org Sat Aug 1 22:29:21 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 22:29:21 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Issue #2103: comments about --source and the PAX issues. Message-ID: <20150801202921.19E4A1C03F4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r625:988a7121a7d5 Date: 2015-08-01 22:30 +0200 http://bitbucket.org/pypy/pypy.org/changeset/988a7121a7d5/ Log: Issue #2103: comments about --source and the PAX issues. diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -282,6 +282,13 @@ PYPY_GC_MAX_DELTA=200MB pypy --jit loop_longevity=300 ../../rpython/bin/rpython -Ojit targetpypystandalone +

  • You can run translations with --source, which only builds the C +source files (and prints at the end where). Then you can cd there +and execute make. This is another way to reduce memory usage. +Note that afterwards, you have to run manually pypy-c +.../pypy/tool/build_cffi_imports.py if you want to be able to import +the cffi-based modules.

    +

  • On Linux, because of asmgcroot, compiling the generated C files is delicate. It requires using gcc with no particularly fancy options. It does not work e.g. with clang, or if you pass uncommon @@ -290,6 +297,12 @@ the shadow stack option instead (for a small performance price in non-JITted code).

    • +

  • Like other JITs, PyPy doesn't work out of the box on some Linux +distributions that trade full POSIX compliance for extra security +features. E.g. with PAX, you have to run PyPy with paxctl -cm. +This also applies to translation (unless you use CPython to run the +translation and you specify --source).

    +
    • diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -295,6 +295,13 @@ PYPY_GC_MAX_DELTA=200MB pypy --jit loop_longevity=300 ../../rpython/bin/rpython -Ojit targetpypystandalone +* You can run translations with ``--source``, which only builds the C + source files (and prints at the end where). Then you can ``cd`` there + and execute ``make``. This is another way to reduce memory usage. + Note that afterwards, you have to run manually ``pypy-c + .../pypy/tool/build_cffi_imports.py`` if you want to be able to import + the cffi-based modules. + * On Linux, because of ``asmgcroot``, compiling the generated C files is delicate. It requires using gcc with no particularly fancy options. It does not work e.g. with clang, or if you pass uncommon @@ -303,6 +310,12 @@ the `shadow stack`_ option instead (for a small performance price in non-JITted code). +* Like other JITs, PyPy doesn't work out of the box on some Linux + distributions that trade full POSIX compliance for extra security + features. E.g. with PAX, you have to run PyPy with ``paxctl -cm``. + This also applies to translation (unless you use CPython to run the + translation and you specify ``--source``). + .. _`x86 (IA-32)`: http://en.wikipedia.org/wiki/IA-32 .. _`x86-64`: http://en.wikipedia.org/wiki/X86-64 .. _SSE2: http://en.wikipedia.org/wiki/SSE2 From noreply at buildbot.pypy.org Sat Aug 1 22:34:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 22:34:53 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Remove the mention of CPython 2.5 Message-ID: <20150801203453.B74E11C03F4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r626:c1dd098ecc66 Date: 2015-08-01 22:35 +0200 http://bitbucket.org/pypy/pypy.org/changeset/c1dd098ecc66/ Log: Remove the mention of CPython 2.5 diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -270,7 +270,7 @@

  • It is recommended to use PyPy to do translations, instead of using CPython, because it is twice as fast. You should just start by downloading an official release of PyPy (with the JIT). If you really have to use CPython -then note that we are talking about CPython 2.5-2.7 here, not CPython 3.x.

    +then note that we are talking about CPython 2.6 or 2.7 here, not CPython 3.x.

  • If RAM usage is a problem (or if you are on Windows, because win32's limit is 2 GB unless you have a 64 bit OS), then you can (for now) tweak some parameters diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -284,7 +284,7 @@ * It is recommended to use PyPy to do translations, instead of using CPython, because it is twice as fast. You should just start by downloading an official release of PyPy (with the JIT). If you really have to use CPython - then note that we are talking about CPython 2.5-2.7 here, not CPython 3.x. + then note that we are talking about CPython 2.6 or 2.7 here, not CPython 3.x. * If RAM usage is a problem (or if you are on Windows, because win32's limit is 2 GB unless you `have a 64 bit OS`_), then you can (for now) tweak some parameters From noreply at buildbot.pypy.org Sat Aug 1 22:36:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 22:36:07 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: Even 2.6's situation is doubtful Message-ID: <20150801203607.1B8A51C03F4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r627:f726dc2ef9f5 Date: 2015-08-01 22:36 +0200 http://bitbucket.org/pypy/pypy.org/changeset/f726dc2ef9f5/ Log: Even 2.6's situation is doubtful diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -270,7 +270,8 @@

  • It is recommended to use PyPy to do translations, instead of using CPython, because it is twice as fast. You should just start by downloading an official release of PyPy (with the JIT). If you really have to use CPython -then note that we are talking about CPython 2.6 or 2.7 here, not CPython 3.x.

    +then note that we are talking about CPython 2.7 here, not CPython 3.x. +(CPython 2.6 might or might not work. Older versions are out.)

  • If RAM usage is a problem (or if you are on Windows, because win32's limit is 2 GB unless you have a 64 bit OS), then you can (for now) tweak some parameters diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -284,7 +284,8 @@ * It is recommended to use PyPy to do translations, instead of using CPython, because it is twice as fast. You should just start by downloading an official release of PyPy (with the JIT). If you really have to use CPython - then note that we are talking about CPython 2.6 or 2.7 here, not CPython 3.x. + then note that we are talking about CPython 2.7 here, not CPython 3.x. + (CPython 2.6 might or might not work. Older versions are out.) * If RAM usage is a problem (or if you are on Windows, because win32's limit is 2 GB unless you `have a 64 bit OS`_), then you can (for now) tweak some parameters From noreply at buildbot.pypy.org Sat Aug 1 22:37:56 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 1 Aug 2015 22:37:56 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: "For a small performance price" -> "For some performance price" Message-ID: <20150801203756.9205C1C03F4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r628:abebd3cb49e8 Date: 2015-08-01 22:38 +0200 http://bitbucket.org/pypy/pypy.org/changeset/abebd3cb49e8/ Log: "For a small performance price" -> "For some performance price" diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -295,7 +295,7 @@ fancy options. It does not work e.g. with clang, or if you pass uncommon options with the CFLAGS environment variable. If you insist on passing these options or using clang, then you can compile PyPy with -the shadow stack option instead (for a small performance price in +the shadow stack option instead (for some performance price in non-JITted code).

  • Like other JITs, PyPy doesn't work out of the box on some Linux diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -308,7 +308,7 @@ fancy options. It does not work e.g. with clang, or if you pass uncommon options with the ``CFLAGS`` environment variable. If you insist on passing these options or using clang, then you can compile PyPy with - the `shadow stack`_ option instead (for a small performance price in + the `shadow stack`_ option instead (for some performance price in non-JITted code). * Like other JITs, PyPy doesn't work out of the box on some Linux From noreply at buildbot.pypy.org Sun Aug 2 12:17:33 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 12:17:33 +0200 (CEST) Subject: [pypy-commit] pypy.org extradoc: update the values Message-ID: <20150802101733.9A3BF1C131A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r629:874a636c48d5 Date: 2015-08-02 12:18 +0200 http://bitbucket.org/pypy/pypy.org/changeset/874a636c48d5/ Log: update the values diff --git a/don1.html b/don1.html --- a/don1.html +++ b/don1.html @@ -9,13 +9,13 @@ - $59988 of $105000 (57.1%) + $60016 of $105000 (57.2%)

    @@ -23,7 +23,7 @@
  • diff --git a/don3.html b/don3.html --- a/don3.html +++ b/don3.html @@ -9,13 +9,13 @@ - $52438 of $60000 (87.4%) + $52486 of $60000 (87.5%)
    @@ -23,7 +23,7 @@
  • From noreply at buildbot.pypy.org Sun Aug 2 15:13:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 15:13:04 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: tweaks Message-ID: <20150802131304.A060A1C0352@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78736:e58e68d58e42 Date: 2015-08-02 10:13 +0200 http://bitbucket.org/pypy/pypy/changeset/e58e68d58e42/ Log: tweaks diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -11,12 +11,13 @@ class VMProf(object): def __init__(self): + "NOT_RPYTHON: use get_vmprof()" self._code_classes = set() self._gather_all_code_objs = lambda: None self._cleanup_() def _cleanup_(self): - self._current_codes = StringBuilder() + self._current_codes = None if sys.maxint == 2147483647: self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit else: @@ -26,11 +27,15 @@ def register_code(self, code, name): """Register the code object. Call when a new code object is made. """ - uid = self._code_unique_id - self._code_unique_id = uid + 4 # so we have two bits to mark stuff + if code._vmprof_unique_id != 0: + return + uid = self._code_unique_id + 4 # so we have two bits to mark stuff code._vmprof_unique_id = uid + self._code_unique_id = uid # b = self._current_codes + if b is None: + b = self._current_codes = StringBuilder() b.append('\x02') write_long_to_string_builder(uid, b) write_long_to_string_builder(len(name), b) @@ -71,16 +76,11 @@ def write_long_to_string_builder(l, b): - if sys.maxint == 2147483647: - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) - else: - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) + b.append(chr(l & 0xff)) + b.append(chr((l >> 8) & 0xff)) + b.append(chr((l >> 16) & 0xff)) + b.append(chr((l >> 24) & 0xff)) + if sys.maxint > 2147483647: b.append(chr((l >> 32) & 0xff)) b.append(chr((l >> 40) & 0xff)) b.append(chr((l >> 48) & 0xff)) From noreply at buildbot.pypy.org Sun Aug 2 15:13:10 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 15:13:10 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: hg merge default Message-ID: <20150802131310.1242C1C0352@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78737:7ea6e6a2f418 Date: 2015-08-02 10:13 +0200 http://bitbucket.org/pypy/pypy/changeset/7ea6e6a2f418/ Log: hg merge default diff too long, truncating to 2000 out of 8977 lines diff --git a/lib-python/2.7/test/test_urllib2.py b/lib-python/2.7/test/test_urllib2.py --- a/lib-python/2.7/test/test_urllib2.py +++ b/lib-python/2.7/test/test_urllib2.py @@ -291,6 +291,7 @@ self.req_headers = [] self.data = None self.raise_on_endheaders = False + self.sock = None self._tunnel_headers = {} def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): diff --git a/lib-python/2.7/urllib2.py b/lib-python/2.7/urllib2.py --- a/lib-python/2.7/urllib2.py +++ b/lib-python/2.7/urllib2.py @@ -1200,6 +1200,12 @@ r = h.getresponse(buffering=True) except TypeError: # buffering kw not supported r = h.getresponse() + # If the server does not send us a 'Connection: close' header, + # HTTPConnection assumes the socket should be left open. Manually + # mark the socket to be closed when this response object goes away. + if h.sock: + h.sock.close() + h.sock = None # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -345,7 +345,10 @@ def _finalize_raw_statement(self, _statement): if self.__rawstatements is not None: - self.__rawstatements.remove(_statement) + try: + self.__rawstatements.remove(_statement) + except KeyError: + return # rare case: already finalized, see issue #2097 _lib.sqlite3_finalize(_statement) def __do_all_statements(self, action, reset_cursors): diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.1.2 +Version: 1.2.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.1.2" -__version_info__ = (1, 1, 2) +__version__ = "1.2.0" +__version_info__ = (1, 2, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -236,6 +236,30 @@ cdecl = self._typeof(cdecl) return self._backend.newp(cdecl, init) + def new_allocator(self, alloc=None, free=None, + should_clear_after_alloc=True): + """Return a new allocator, i.e. a function that behaves like ffi.new() + but uses the provided low-level 'alloc' and 'free' functions. + + 'alloc' is called with the size as argument. If it returns NULL, a + MemoryError is raised. 'free' is called with the result of 'alloc' + as argument. Both can be either Python function or directly C + functions. If 'free' is None, then no free function is called. + If both 'alloc' and 'free' are None, the default is used. + + If 'should_clear_after_alloc' is set to False, then the memory + returned by 'alloc' is assumed to be already cleared (or you are + fine with garbage); otherwise CFFI will clear it. + """ + compiled_ffi = self._backend.FFI() + allocator = compiled_ffi.new_allocator(alloc, free, + should_clear_after_alloc) + def allocate(cdecl, init=None): + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return allocator(cdecl, init) + return allocate + def cast(self, cdecl, source): """Similar to a C cast: returns an instance of the named C type initialized with the given 'source'. The source is @@ -286,7 +310,7 @@ """ return self._backend.from_buffer(self.BCharA, python_buffer) - def callback(self, cdecl, python_callable=None, error=None): + def callback(self, cdecl, python_callable=None, error=None, onerror=None): """Return a callback object or a decorator making such a callback object. 'cdecl' must name a C function pointer type. The callback invokes the specified 'python_callable' (which may @@ -298,7 +322,8 @@ if not callable(python_callable): raise TypeError("the 'python_callable' argument " "is not callable") - return self._backend.callback(cdecl, python_callable, error) + return self._backend.callback(cdecl, python_callable, + error, onerror) if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl, consider_function_as_funcptr=True) if python_callable is None: @@ -327,6 +352,13 @@ data. Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called. """ + try: + gcp = self._backend.gcp + except AttributeError: + pass + else: + return gcp(cdata, destructor) + # with self._lock: try: gc_weakrefs = self.gc_weakrefs @@ -428,6 +460,8 @@ raise TypeError("ffi.include() expects an argument that is also of" " type cffi.FFI, not %r" % ( type(ffi_to_include).__name__,)) + if ffi_to_include is self: + raise ValueError("self.include(self)") with ffi_to_include._lock: with self._lock: self._parser.include(ffi_to_include._parser) diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py --- a/lib_pypy/cffi/backend_ctypes.py +++ b/lib_pypy/cffi/backend_ctypes.py @@ -989,7 +989,8 @@ def cast(self, BType, source): return BType._cast_from(source) - def callback(self, BType, source, error): + def callback(self, BType, source, error, onerror): + assert onerror is None # XXX not implemented return BType(source, error) typeof = type diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py --- a/lib_pypy/cffi/cffi_opcode.py +++ b/lib_pypy/cffi/cffi_opcode.py @@ -53,6 +53,7 @@ OP_GLOBAL_VAR = 33 OP_DLOPEN_FUNC = 35 OP_DLOPEN_CONST = 37 +OP_GLOBAL_VAR_F = 39 PRIM_VOID = 0 PRIM_BOOL = 1 diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -633,6 +633,8 @@ def include(self, other): for name, tp in other._declarations.items(): + if name.startswith('anonymous $enum_$'): + continue # fix for test_anonymous_enum_include kind = name.split(' ', 1)[0] if kind in ('struct', 'union', 'enum', 'anonymous'): self._declare(name, tp, included=True) diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py --- a/lib_pypy/cffi/model.py +++ b/lib_pypy/cffi/model.py @@ -35,9 +35,6 @@ def is_integer_type(self): return False - def sizeof_enabled(self): - return False - def get_cached_btype(self, ffi, finishlist, can_delay=False): try: BType = ffi._cached_btypes[self] @@ -80,8 +77,7 @@ class BasePrimitiveType(BaseType): - def sizeof_enabled(self): - return True + pass class PrimitiveType(BasePrimitiveType): @@ -205,9 +201,6 @@ class FunctionPtrType(BaseFunctionType): _base_pattern = '(*&)(%s)' - def sizeof_enabled(self): - return True - def build_backend_type(self, ffi, finishlist): result = self.result.get_cached_btype(ffi, finishlist) args = [] @@ -233,9 +226,6 @@ extra = self._base_pattern self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) - def sizeof_enabled(self): - return True - def build_backend_type(self, ffi, finishlist): BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True) return global_cache(self, ffi, 'new_pointer_type', BItem) @@ -276,9 +266,6 @@ self.c_name_with_marker = ( self.item.c_name_with_marker.replace('&', brackets)) - def sizeof_enabled(self): - return self.item.sizeof_enabled() and self.length is not None - def resolve_length(self, newlength): return ArrayType(self.item, newlength) @@ -433,9 +420,6 @@ from . import ffiplatform raise ffiplatform.VerificationMissing(self._get_c_name()) - def sizeof_enabled(self): - return self.fldtypes is not None - def build_backend_type(self, ffi, finishlist): self.check_not_partial() finishlist.append(self) @@ -464,9 +448,6 @@ self.baseinttype = baseinttype self.build_c_name_with_marker() - def sizeof_enabled(self): - return True # not strictly true, but external enums are obscure - def force_the_name(self, forcename): StructOrUnionOrEnum.force_the_name(self, forcename) if self.forcename is None: diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h --- a/lib_pypy/cffi/parse_c_type.h +++ b/lib_pypy/cffi/parse_c_type.h @@ -26,6 +26,7 @@ #define _CFFI_OP_GLOBAL_VAR 33 #define _CFFI_OP_DLOPEN_FUNC 35 #define _CFFI_OP_DLOPEN_CONST 37 +#define _CFFI_OP_GLOBAL_VAR_F 39 #define _CFFI_PRIM_VOID 0 #define _CFFI_PRIM_BOOL 1 diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -981,10 +981,6 @@ if not self.target_is_python and tp.is_integer_type(): type_op = CffiOp(OP_CONSTANT_INT, -1) else: - if not tp.sizeof_enabled(): - raise ffiplatform.VerificationError( - "constant '%s' is of type '%s', whose size is not known" - % (name, tp._get_c_name())) if self.target_is_python: const_kind = OP_DLOPEN_CONST else: @@ -1069,18 +1065,36 @@ self._do_collect_type(self._global_type(tp, name)) def _generate_cpy_variable_decl(self, tp, name): - pass + prnt = self._prnt + tp = self._global_type(tp, name) + if isinstance(tp, model.ArrayType) and tp.length is None: + tp = tp.item + ampersand = '' + else: + ampersand = '&' + # This code assumes that casts from "tp *" to "void *" is a + # no-op, i.e. a function that returns a "tp *" can be called + # as if it returned a "void *". This should be generally true + # on any modern machine. The only exception to that rule (on + # uncommon architectures, and as far as I can tell) might be + # if 'tp' were a function type, but that is not possible here. + # (If 'tp' is a function _pointer_ type, then casts from "fn_t + # **" to "void *" are again no-ops, as far as I can tell.) + prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,))) + prnt('{') + prnt(' return %s(%s);' % (ampersand, name)) + prnt('}') + prnt() def _generate_cpy_variable_ctx(self, tp, name): tp = self._global_type(tp, name) type_index = self._typesdict[tp] - type_op = CffiOp(OP_GLOBAL_VAR, type_index) - if tp.sizeof_enabled(): - size = "sizeof(%s)" % (name,) + if self.target_is_python: + op = OP_GLOBAL_VAR else: - size = 0 + op = OP_GLOBAL_VAR_F self._lsts["global"].append( - GlobalExpr(name, '&%s' % name, type_op, size)) + GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index))) # ---------- # emitting the opcodes for individual types diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -23,14 +23,14 @@ default_modules.update([ "_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "cmath", "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast", - "_io", "_random", "__pypy__", "_testing" + "_io", "_random", "__pypy__", "_testing", "time" ]) # --allworkingmodules working_modules = default_modules.copy() working_modules.update([ - "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" , + "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios", "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses", "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -70,6 +70,20 @@ .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv +Module xyz does not work in the sandboxed PyPy? +----------------------------------------------- + +You cannot import *any* extension module in a `sandboxed PyPy`_, +sorry. Even the built-in modules available are very limited. +Sandboxing in PyPy is a good proof of concept, really safe IMHO, but +it is only a proof of concept. It seriously requires someone working +on it. Before this occurs, it can only be used it for "pure Python" +examples: programs that import mostly nothing (or only pure Python +modules, recursively). + +.. _`sandboxed PyPy`: sandbox.html + + .. _`See below.`: Do CPython Extension modules work with PyPy? diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst --- a/pypy/doc/sandbox.rst +++ b/pypy/doc/sandbox.rst @@ -103,12 +103,15 @@ Howto ----- -In pypy/goal:: +Grab a copy of the pypy repository_. In the directory pypy/goal, run:: ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py If you don't have a regular PyPy installed, you should, because it's -faster to translate, but you can also run ``python translate.py`` instead. +faster to translate; but you can also run the same line with ``python`` +in front. + +.. _repository: https://bitbucket.org/pypy/pypy To run it, use the tools in the pypy/sandbox directory:: @@ -136,8 +139,6 @@ Not all operations are supported; e.g. if you type os.readlink('...'), the controller crashes with an exception and the subprocess is killed. Other operations make the subprocess die directly with a "Fatal RPython -error". None of this is a security hole; it just means that if you try -to run some random program, it risks getting killed depending on the -Python built-in functions it tries to call. This is a matter of the -sandboxing layer being incomplete so far, but it should not really be -a problem in practice. +error". None of this is a security hole. More importantly, *most other +built-in modules are not enabled. Please read all the warnings in this +page before complaining about this. Contributions welcome.* diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -20,5 +20,38 @@ .. branch: run-create_cffi_imports Build cffi import libraries as part of translation by monkey-patching an -aditional task into translation +additional task into translation +.. branch: int-float-list-strategy + +Use a compact strategy for Python lists that mix integers and floats, +at least if the integers fit inside 32 bits. These lists are now +stored as an array of floats, like lists that contain only floats; the +difference is that integers are stored as tagged NaNs. (This should +have no visible effect! After ``lst = [42, 42.5]``, the value of +``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.) + +.. branch: cffi-callback-onerror +.. branch: cffi-new-allocator + +.. branch: unicode-dtype + +Partial implementation of unicode dtype and unicode scalars. + +.. branch: dtypes-compatability + +Improve compatibility with numpy dtypes; handle offsets to create unions, +fix str() and repr(), allow specifying itemsize, metadata and titles, add flags, +allow subclassing dtype + +.. branch: indexing + +Refactor array indexing to support ellipses. + +.. branch: numpy-docstrings + +Allow the docstrings of built-in numpy objects to be set at run-time. + +.. branch: nditer-revisited + +Implement nditer 'buffered' flag and fix some edge cases diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -297,7 +297,12 @@ options = make_dict(config) wrapstr = 'space.wrap(%r)' % (options) pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr + if config.objspace.usemodules._cffi_backend: + self.hack_for_cffi_modules(driver) + return self.get_entry_point(config) + + def hack_for_cffi_modules(self, driver): # HACKHACKHACK # ugly hack to modify target goal from compile_c to build_cffi_imports # this should probably get cleaned up and merged with driver.create_exe @@ -336,8 +341,6 @@ driver.default_goal = 'build_cffi_imports' # HACKHACKHACK end - return self.get_entry_point(config) - def jitpolicy(self, driver): from pypy.module.pypyjit.policy import PyPyJitPolicy from pypy.module.pypyjit.hooks import pypy_hooks diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -200,7 +200,7 @@ w_result = space.get_and_call_function(w_impl, self) if space.isinstance_w(w_result, space.w_buffer): return w_result.buffer_w(space, flags) - raise TypeError + raise BufferInterfaceNotFound def readbuf_w(self, space): w_impl = space.lookup(self, '__buffer__') @@ -208,7 +208,7 @@ w_result = space.get_and_call_function(w_impl, self) if space.isinstance_w(w_result, space.w_buffer): return w_result.readbuf_w(space) - raise TypeError + raise BufferInterfaceNotFound def writebuf_w(self, space): w_impl = space.lookup(self, '__buffer__') @@ -216,7 +216,7 @@ w_result = space.get_and_call_function(w_impl, self) if space.isinstance_w(w_result, space.w_buffer): return w_result.writebuf_w(space) - raise TypeError + raise BufferInterfaceNotFound def charbuf_w(self, space): w_impl = space.lookup(self, '__buffer__') @@ -224,7 +224,7 @@ w_result = space.get_and_call_function(w_impl, self) if space.isinstance_w(w_result, space.w_buffer): return w_result.charbuf_w(space) - raise TypeError + raise BufferInterfaceNotFound def str_w(self, space): self._typed_unwrap_error(space, "string") @@ -354,6 +354,9 @@ class DescrMismatch(Exception): pass +class BufferInterfaceNotFound(Exception): + pass + def wrappable_class_name(Class): try: return Class.typedef.name @@ -1403,7 +1406,7 @@ # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer) try: return w_obj.buffer_w(self, flags) - except TypeError: + except BufferInterfaceNotFound: raise oefmt(self.w_TypeError, "'%T' does not have the buffer interface", w_obj) @@ -1411,7 +1414,7 @@ # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer) try: return w_obj.readbuf_w(self) - except TypeError: + except BufferInterfaceNotFound: raise oefmt(self.w_TypeError, "expected a readable buffer object") @@ -1419,7 +1422,7 @@ # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer) try: return w_obj.writebuf_w(self) - except TypeError: + except BufferInterfaceNotFound: raise oefmt(self.w_TypeError, "expected a writeable buffer object") @@ -1427,7 +1430,7 @@ # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer) try: return w_obj.charbuf_w(self) - except TypeError: + except BufferInterfaceNotFound: raise oefmt(self.w_TypeError, "expected a character buffer object") @@ -1451,11 +1454,11 @@ return self.str(w_obj).readbuf_w(self) try: return w_obj.buffer_w(self, 0) - except TypeError: + except BufferInterfaceNotFound: pass try: return w_obj.readbuf_w(self) - except TypeError: + except BufferInterfaceNotFound: self._getarg_error("string or buffer", w_obj) elif code == 's#': if self.isinstance_w(w_obj, self.w_str): @@ -1464,24 +1467,23 @@ return self.str(w_obj).str_w(self) try: return w_obj.readbuf_w(self).as_str() - except TypeError: + except BufferInterfaceNotFound: self._getarg_error("string or read-only buffer", w_obj) elif code == 'w*': try: - try: - return w_obj.buffer_w(self, self.BUF_WRITABLE) - except OperationError: - self._getarg_error("read-write buffer", w_obj) - except TypeError: + return w_obj.buffer_w(self, self.BUF_WRITABLE) + except OperationError: + self._getarg_error("read-write buffer", w_obj) + except BufferInterfaceNotFound: pass try: return w_obj.writebuf_w(self) - except TypeError: + except BufferInterfaceNotFound: self._getarg_error("read-write buffer", w_obj) elif code == 't#': try: return w_obj.charbuf_w(self) - except TypeError: + except BufferInterfaceNotFound: self._getarg_error("string or read-only character buffer", w_obj) else: assert False @@ -1503,13 +1505,13 @@ raise try: buf = w_obj.buffer_w(self, 0) - except TypeError: + except BufferInterfaceNotFound: pass else: return buf.as_str() try: buf = w_obj.readbuf_w(self) - except TypeError: + except BufferInterfaceNotFound: self._getarg_error("string or buffer", w_obj) else: return buf.as_str() diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -252,7 +252,8 @@ w_t, w_v, w_tb], """(where, objrepr, extra_line, t, v, tb): import sys, traceback - sys.stderr.write('From %s%s:\\n' % (where, objrepr)) + if where or objrepr: + sys.stderr.write('From %s%s:\\n' % (where, objrepr)) if extra_line: sys.stderr.write(extra_line) traceback.print_exception(t, v, tb) diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -15,7 +15,10 @@ self.running = False def descr__repr__(self, space): - code_name = self.pycode.co_name + if self.pycode is None: + code_name = '' + else: + code_name = self.pycode.co_name addrstring = self.getaddrstring(space) return space.wrap("" % (code_name, addrstring)) @@ -45,6 +48,8 @@ w_framestate, w_running = args_w if space.is_w(w_framestate, space.w_None): self.frame = None + self.space = space + self.pycode = None else: frame = instantiate(space.FrameClass) # XXX fish frame.descr__setstate__(space, w_framestate) @@ -62,9 +67,10 @@ def send_ex(self, w_arg, operr=None): pycode = self.pycode - if jit.we_are_jitted() and should_not_inline(pycode): - generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg, - operr=operr, pycode=pycode) + if pycode is not None: + if jit.we_are_jitted() and should_not_inline(pycode): + generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg, + operr=operr, pycode=pycode) return self._send_ex(w_arg, operr) def _send_ex(self, w_arg, operr): @@ -158,7 +164,10 @@ return self.pycode def descr__name__(self, space): - code_name = self.pycode.co_name + if self.pycode is None: + code_name = '' + else: + code_name = self.pycode.co_name return space.wrap(code_name) # Results can be either an RPython list of W_Root, or it can be an diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -711,11 +711,17 @@ w_item = self.popvalue() if self.space.is_w(w_stream, self.space.w_None): w_stream = sys_stdout(self.space) # grumble grumble special cases - print_item_to(self.space, w_item, w_stream) + print_item_to(self.space, self._printable_object(w_item), w_stream) def PRINT_ITEM(self, oparg, next_instr): w_item = self.popvalue() - print_item(self.space, w_item) + print_item(self.space, self._printable_object(w_item)) + + def _printable_object(self, w_obj): + space = self.space + if not space.isinstance_w(w_obj, space.w_unicode): + w_obj = space.str(w_obj) + return w_obj def PRINT_NEWLINE_TO(self, oparg, next_instr): w_stream = self.popvalue() @@ -1535,9 +1541,9 @@ stream.write(" ") # give to write() an argument which is either a string or a unicode - # (and let it deals itself with unicode handling) - if not isinstance(x, unicode): - x = str(x) + # (and let it deals itself with unicode handling). The check "is + # unicode" should not use isinstance() at app-level, because that + # could be fooled by strange objects, so it is done at interp-level. stream.write(x) # add a softspace unless we just printed a string which ends in a '\t' diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py --- a/pypy/interpreter/test/test_interpreter.py +++ b/pypy/interpreter/test/test_interpreter.py @@ -299,6 +299,30 @@ finally: sys.stdout = save + def test_print_strange_object(self): + import sys + + class A(object): + def __getattribute__(self, name): + print "seeing", name + def __str__(self): + return 'A!!' + save = sys.stdout + class Out(object): + def __init__(self): + self.data = [] + def write(self, x): + self.data.append((type(x), x)) + sys.stdout = out = Out() + try: + a = A() + assert out.data == [] + print a + assert out.data == [(str, 'A!!'), + (str, '\n')] + finally: + sys.stdout = save + def test_identity(self): def f(x): return x assert f(666) == 666 diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -491,6 +491,22 @@ assert pack.mod is result + def test_pickle_generator_crash(self): + import pickle + + def f(): + yield 0 + + x = f() + x.next() + try: + x.next() + except StopIteration: + y = pickle.loads(pickle.dumps(x)) + assert 'finished' in y.__name__ + assert 'finished' in repr(y) + assert y.gi_code is None + class AppTestGeneratorCloning: def setup_class(cls): diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py --- a/pypy/module/__builtin__/app_functional.py +++ b/pypy/module/__builtin__/app_functional.py @@ -53,6 +53,33 @@ last = last + x return last + +class _Cons(object): + def __init__(self, prev, iter): + self.prev = prev + self.iter = iter + + def fetch(self): + # recursive, loop-less version of the algorithm: works best for a + # fixed number of "collections" in the call to map(func, *collections) + prev = self.prev + if prev is None: + args1 = () + stop = True + else: + args1, stop = prev.fetch() + iter = self.iter + if iter is None: + val = None + else: + try: + val = next(iter) + stop = False + except StopIteration: + self.iter = None + val = None + return args1 + (val,), stop + def map(func, *collections): """map(function, sequence[, sequence, ...]) -> list @@ -69,45 +96,30 @@ if num_collections == 1: if none_func: return list(collections[0]) - # Special case for the really common case of a single collection, - # this can be eliminated if we could unroll that loop that creates - # `args` based on whether or not len(collections) was constant + # Special case for the really common case of a single collection seq = collections[0] with _ManagedNewlistHint(operator._length_hint(seq, 0)) as result: for item in seq: result.append(func(item)) return result - # Gather the iterators (pair of (iter, has_finished)) and guess the + # Gather the iterators into _Cons objects and guess the # result length (the max of the input lengths) - iterators = [] + c = None max_hint = 0 for seq in collections: - iterators.append((iter(seq), False)) + c = _Cons(c, iter(seq)) max_hint = max(max_hint, operator._length_hint(seq, 0)) with _ManagedNewlistHint(max_hint) as result: while True: - cont = False - args = [] - for idx, (iterator, has_finished) in enumerate(iterators): - val = None - if not has_finished: - try: - val = next(iterator) - except StopIteration: - iterators[idx] = (None, True) - else: - cont = True - args.append(val) - args = tuple(args) - if cont: - if none_func: - result.append(args) - else: - result.append(func(*args)) + args, stop = c.fetch() + if stop: + return result + if none_func: + result.append(args) else: - return result + result.append(func(*args)) class _ManagedNewlistHint(object): """ Context manager returning a newlist_hint upon entry. diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py --- a/pypy/module/__builtin__/test/test_abstractinst.py +++ b/pypy/module/__builtin__/test/test_abstractinst.py @@ -202,3 +202,17 @@ __subclass__ = set([int]) assert issubclass(int, Integer) assert issubclass(int, (Integer,)) + + def test_dont_call_instancecheck_fast_path(self): + called = [] + + class M(type): + def __instancecheck__(self, obj): + called.append("called") + + class C: + __metaclass__ = M + + c = C() + assert isinstance(c, C) + assert not called diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -57,6 +57,11 @@ b = [] assert map(lambda x, y: x, a, b) == a + def test_map_second_item(self): + a = [] + b = [1, 2, 3, 4, 5] + assert map(lambda x, y: y, a, b) == b + def test_map_iterables(self): class A(object): def __init__(self, n): diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload -VERSION = "1.1.2" +VERSION = "1.2.0" class Module(MixedModule): diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py new file mode 100644 --- /dev/null +++ b/pypy/module/_cffi_backend/allocator.py @@ -0,0 +1,86 @@ +from pypy.interpreter.error import oefmt +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import TypeDef +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault + +from rpython.rtyper.lltypesystem import lltype, rffi + + +class W_Allocator(W_Root): + _immutable_ = True + + def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc): + self.ffi = ffi # may be None + self.w_alloc = w_alloc + self.w_free = w_free + self.should_clear_after_alloc = should_clear_after_alloc + + def allocate(self, space, datasize, ctype, length=-1): + from pypy.module._cffi_backend import cdataobj, ctypeptr + if self.w_alloc is None: + if self.should_clear_after_alloc: + ptr = lltype.malloc(rffi.CCHARP.TO, datasize, + flavor='raw', zero=True) + else: + ptr = lltype.malloc(rffi.CCHARP.TO, datasize, + flavor='raw', zero=False) + return cdataobj.W_CDataNewStd(space, ptr, ctype, length) + else: + w_raw_cdata = space.call_function(self.w_alloc, + space.wrap(datasize)) + if not isinstance(w_raw_cdata, cdataobj.W_CData): + raise oefmt(space.w_TypeError, + "alloc() must return a cdata object (got %T)", + w_raw_cdata) + if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray): + raise oefmt(space.w_TypeError, + "alloc() must return a cdata pointer, not '%s'", + w_raw_cdata.ctype.name) + # + ptr = w_raw_cdata.unsafe_escaping_ptr() + if not ptr: + raise oefmt(space.w_MemoryError, "alloc() returned NULL") + # + if self.should_clear_after_alloc: + rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0, + rffi.cast(rffi.SIZE_T, datasize)) + # + if self.w_free is None: + # use this class which does not have a __del__, but still + # keeps alive w_raw_cdata + res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length) + else: + res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length) + res.w_free = self.w_free + res.w_raw_cdata = w_raw_cdata + return res + + @unwrap_spec(w_init=WrappedDefault(None)) + def descr_call(self, space, w_arg, w_init): + ffi = self.ffi + assert ffi is not None + w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE) + return w_ctype.newp(w_init, self) + + +W_Allocator.typedef = TypeDef( + 'FFIAllocator', + __call__ = interp2app(W_Allocator.descr_call), + ) +W_Allocator.typedef.acceptable_as_base_class = False + + +def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc): + space = ffi.space + if space.is_none(w_alloc): + w_alloc = None + if space.is_none(w_free): + w_free = None + if w_alloc is None and w_free is not None: + raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'") + alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc)) + return space.wrap(alloc) + + +default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True) +nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -22,8 +22,9 @@ class W_CDataCallback(W_CData): #_immutable_fields_ = ... ll_error = lltype.nullptr(rffi.CCHARP.TO) + w_onerror = None - def __init__(self, space, ctype, w_callable, w_error): + def __init__(self, space, ctype, w_callable, w_error, w_onerror): raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc()) W_CData.__init__(self, space, raw_closure, ctype) # @@ -31,6 +32,12 @@ raise oefmt(space.w_TypeError, "expected a callable object, not %T", w_callable) self.w_callable = w_callable + if not space.is_none(w_onerror): + if not space.is_true(space.callable(w_onerror)): + raise oefmt(space.w_TypeError, + "expected a callable object for 'onerror', not %T", + w_onerror) + self.w_onerror = w_onerror # fresult = self.getfunctype().ctitem size = fresult.size @@ -161,6 +168,29 @@ STDERR = 2 + at jit.dont_look_inside +def _handle_applevel_exception(space, callback, e, ll_res, extra_line): + callback.write_error_return_value(ll_res) + if callback.w_onerror is None: + callback.print_error(e, extra_line) + else: + try: + e.normalize_exception(space) + w_t = e.w_type + w_v = e.get_w_value(space) + w_tb = space.wrap(e.get_traceback()) + w_res = space.call_function(callback.w_onerror, + w_t, w_v, w_tb) + if not space.is_none(w_res): + callback.convert_result(ll_res, w_res) + except OperationError, e2: + # double exception! print a double-traceback... + callback.print_error(e, extra_line) # original traceback + e2.write_unraisable(space, '', with_traceback=True, + extra_line="\nDuring the call to 'onerror', " + "another exception occurred:\n\n") + + @jit.jit_callback("CFFI") def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. @@ -178,7 +208,7 @@ try: os.write(STDERR, "SystemError: invoking a callback " "that was already freed\n") - except OSError: + except: pass # In this case, we don't even know how big ll_res is. Let's assume # it is just a 'ffi_arg', and store 0 there. @@ -195,9 +225,7 @@ extra_line = "Trying to convert the result back to C:\n" callback.convert_result(ll_res, w_res) except OperationError, e: - # got an app-level exception - callback.print_error(e, extra_line) - callback.write_error_return_value(ll_res) + _handle_applevel_exception(space, callback, e, ll_res, extra_line) # except Exception, e: # oups! last-level attempt to recover. @@ -205,7 +233,7 @@ os.write(STDERR, "SystemError: callback raised ") os.write(STDERR, str(e)) os.write(STDERR, "\n") - except OSError: + except: pass callback.write_error_return_value(ll_res) if must_leave: diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -363,16 +363,19 @@ def _sizeof(self): return self.ctype.size + def with_gc(self, w_destructor): + with self as ptr: + return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor) + class W_CDataMem(W_CData): - """This is the base class used for cdata objects that own and free - their memory. Used directly by the results of cffi.cast('int', x) - or other primitive explicitly-casted types. It is further subclassed - by W_CDataNewOwning.""" + """This is used only by the results of cffi.cast('int', x) + or other primitive explicitly-casted types.""" _attrs_ = [] - def __init__(self, space, size, ctype): - cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True) + def __init__(self, space, ctype): + cdata = lltype.malloc(rffi.CCHARP.TO, ctype.size, flavor='raw', + zero=False) W_CData.__init__(self, space, cdata, ctype) @rgc.must_be_light_finalizer @@ -380,36 +383,65 @@ lltype.free(self._ptr, flavor='raw') -class W_CDataNewOwning(W_CDataMem): - """This is the class used for the cata objects created by newp().""" - _attrs_ = [] +class W_CDataNewOwning(W_CData): + """This is the abstract base class used for cdata objects created + by newp(). They create and free their own memory according to an + allocator.""" + + # the 'length' is either >= 0 for arrays, or -1 for pointers. + _attrs_ = ['length'] + _immutable_fields_ = ['length'] + + def __init__(self, space, cdata, ctype, length=-1): + W_CData.__init__(self, space, cdata, ctype) + self.length = length def _repr_extra(self): return self._repr_extra_owning() - -class W_CDataNewOwningLength(W_CDataNewOwning): - """Subclass with an explicit length, for allocated instances of - the C type 'foo[]'.""" - _attrs_ = ['length'] - _immutable_fields_ = ['length'] - - def __init__(self, space, size, ctype, length): - W_CDataNewOwning.__init__(self, space, size, ctype) - self.length = length - def _sizeof(self): - from pypy.module._cffi_backend import ctypearray ctype = self.ctype - assert isinstance(ctype, ctypearray.W_CTypeArray) - return self.length * ctype.ctitem.size + if self.length >= 0: + from pypy.module._cffi_backend import ctypearray + assert isinstance(ctype, ctypearray.W_CTypeArray) + return self.length * ctype.ctitem.size + else: + return ctype.size def get_array_length(self): return self.length +class W_CDataNewStd(W_CDataNewOwning): + """Subclass using the standard allocator, lltype.malloc()/lltype.free()""" + _attrs_ = [] + + @rgc.must_be_light_finalizer + def __del__(self): + lltype.free(self._ptr, flavor='raw') + + +class W_CDataNewNonStdNoFree(W_CDataNewOwning): + """Subclass using a non-standard allocator, no free()""" + _attrs_ = ['w_raw_cdata'] + +class W_CDataNewNonStdFree(W_CDataNewNonStdNoFree): + """Subclass using a non-standard allocator, with a free()""" + _attrs_ = ['w_free'] + + def __del__(self): + self.clear_all_weakrefs() + self.enqueue_for_destruction(self.space, + W_CDataNewNonStdFree.call_destructor, + 'destructor of ') + + def call_destructor(self): + assert isinstance(self, W_CDataNewNonStdFree) + self.space.call_function(self.w_free, self.w_raw_cdata) + + class W_CDataPtrToStructOrUnion(W_CData): - """This subclass is used for the pointer returned by new('struct foo'). + """This subclass is used for the pointer returned by new('struct foo *'). It has a strong reference to a W_CDataNewOwning that really owns the struct, which is the object returned by the app-level expression 'p[0]'. But it is not itself owning any memory, although its repr says so; @@ -483,6 +515,26 @@ self.length, self.space.type(self.w_keepalive).name) +class W_CDataGCP(W_CData): + """For ffi.gc().""" + _attrs_ = ['w_original_cdata', 'w_destructor'] + _immutable_fields_ = ['w_original_cdata', 'w_destructor'] + + def __init__(self, space, cdata, ctype, w_original_cdata, w_destructor): + W_CData.__init__(self, space, cdata, ctype) + self.w_original_cdata = w_original_cdata + self.w_destructor = w_destructor + + def __del__(self): + self.clear_all_weakrefs() + self.enqueue_for_destruction(self.space, W_CDataGCP.call_destructor, + 'destructor of ') + + def call_destructor(self): + assert isinstance(self, W_CDataGCP) + self.space.call_function(self.w_destructor, self.w_original_cdata) + + W_CData.typedef = TypeDef( '_cffi_backend.CData', __module__ = '_cffi_backend', diff --git a/pypy/module/_cffi_backend/cdlopen.py b/pypy/module/_cffi_backend/cdlopen.py --- a/pypy/module/_cffi_backend/cdlopen.py +++ b/pypy/module/_cffi_backend/cdlopen.py @@ -36,7 +36,10 @@ self.libname) try: cdata = dlsym(self.libhandle, name) + found = bool(cdata) except KeyError: + found = False + if not found: raise oefmt(self.ffi.w_FFIError, "symbol '%s' not found in library '%s'", name, self.libname) diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py --- a/pypy/module/_cffi_backend/cffi_opcode.py +++ b/pypy/module/_cffi_backend/cffi_opcode.py @@ -53,6 +53,7 @@ OP_GLOBAL_VAR = 33 OP_DLOPEN_FUNC = 35 OP_DLOPEN_CONST = 37 +OP_GLOBAL_VAR_F = 39 PRIM_VOID = 0 PRIM_BOOL = 1 diff --git a/pypy/module/_cffi_backend/cgc.py b/pypy/module/_cffi_backend/cgc.py deleted file mode 100644 --- a/pypy/module/_cffi_backend/cgc.py +++ /dev/null @@ -1,29 +0,0 @@ -from rpython.rlib import jit - - - at jit.dont_look_inside -def gc_weakrefs_build(ffi, w_cdata, w_destructor): - from pypy.module._weakref import interp__weakref - - space = ffi.space - if ffi.w_gc_wref_remove is None: - ffi.gc_wref_dict = {} - ffi.w_gc_wref_remove = space.getattr(space.wrap(ffi), - space.wrap("__gc_wref_remove")) - - w_new_cdata = w_cdata.ctype.cast(w_cdata) - assert w_new_cdata is not w_cdata - - w_ref = interp__weakref.make_weakref_with_callback( - space, - space.gettypefor(interp__weakref.W_Weakref), - w_new_cdata, - ffi.w_gc_wref_remove) - - ffi.gc_wref_dict[w_ref] = (w_destructor, w_cdata) - return w_new_cdata - - -def gc_wref_remove(ffi, w_ref): - (w_destructor, w_cdata) = ffi.gc_wref_dict.pop(w_ref) - ffi.space.call_function(w_destructor, w_cdata) diff --git a/pypy/module/_cffi_backend/cglob.py b/pypy/module/_cffi_backend/cglob.py --- a/pypy/module/_cffi_backend/cglob.py +++ b/pypy/module/_cffi_backend/cglob.py @@ -1,24 +1,66 @@ +from pypy.interpreter.error import oefmt from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.typedef import TypeDef from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend import newtype +from rpython.rlib.objectmodel import we_are_translated +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo class W_GlobSupport(W_Root): - def __init__(self, space, w_ctype, ptr): + _immutable_fields_ = ['w_ctype', 'ptr', 'fetch_addr'] + + def __init__(self, space, name, w_ctype, ptr=lltype.nullptr(rffi.CCHARP.TO), + fetch_addr=lltype.nullptr(rffi.VOIDP.TO)): self.space = space + self.name = name self.w_ctype = w_ctype self.ptr = ptr + self.fetch_addr = fetch_addr + + def fetch_global_var_addr(self): + if self.ptr: + result = self.ptr + else: + if not we_are_translated(): + FNPTR = rffi.CCallback([], rffi.VOIDP) + fetch_addr = rffi.cast(FNPTR, self.fetch_addr) + result = fetch_addr() + else: + # careful in translated versions: we need to call fetch_addr, + # but in a GIL-releasing way. The easiest is to invoke a + # llexternal() helper. + result = pypy__cffi_fetch_var(self.fetch_addr) + result = rffi.cast(rffi.CCHARP, result) + if not result: + from pypy.module._cffi_backend import ffi_obj + ffierror = ffi_obj.get_ffi_error(self.space) + raise oefmt(ffierror, "global variable '%s' is at address NULL", + self.name) + return result def read_global_var(self): - return self.w_ctype.convert_to_object(self.ptr) + return self.w_ctype.convert_to_object(self.fetch_global_var_addr()) def write_global_var(self, w_newvalue): - self.w_ctype.convert_from_object(self.ptr, w_newvalue) + self.w_ctype.convert_from_object(self.fetch_global_var_addr(), + w_newvalue) def address(self): w_ctypeptr = newtype.new_pointer_type(self.space, self.w_ctype) - return W_CData(self.space, self.ptr, w_ctypeptr) + return W_CData(self.space, self.fetch_global_var_addr(), w_ctypeptr) W_GlobSupport.typedef = TypeDef("FFIGlobSupport") W_GlobSupport.typedef.acceptable_as_base_class = False + + +eci = ExternalCompilationInfo(post_include_bits=[""" +static void *pypy__cffi_fetch_var(void *fetch_addr) { + return ((void*(*)(void))fetch_addr)(); +} +"""]) + +pypy__cffi_fetch_var = rffi.llexternal( + "pypy__cffi_fetch_var", [rffi.VOIDP], rffi.VOIDP, + compilation_info=eci) diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -28,7 +28,7 @@ def _alignof(self): return self.ctitem.alignof() - def newp(self, w_init): + def newp(self, w_init, allocator): space = self.space datasize = self.size # @@ -40,12 +40,10 @@ except OverflowError: raise OperationError(space.w_OverflowError, space.wrap("array size would overflow a ssize_t")) - # - cdata = cdataobj.W_CDataNewOwningLength(space, datasize, - self, length) + else: + length = self.length # - else: - cdata = cdataobj.W_CDataNewOwning(space, datasize, self) + cdata = allocator.allocate(space, datasize, self, length) # if not space.is_w(w_init, space.w_None): with cdata as ptr: diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -55,7 +55,7 @@ def pack_list_of_items(self, cdata, w_ob): return False - def newp(self, w_init): + def newp(self, w_init, allocator): space = self.space raise oefmt(space.w_TypeError, "expected a pointer or array ctype, got '%s'", self.name) @@ -90,6 +90,16 @@ def _convert_error(self, expected, w_got): space = self.space if isinstance(w_got, cdataobj.W_CData): + if self.name == w_got.ctype.name: + # in case we'd give the error message "initializer for + # ctype 'A' must be a pointer to same type, not cdata + # 'B'", but with A=B, then give instead a different error + # message to try to clear up the confusion + return oefmt(space.w_TypeError, + "initializer for ctype '%s' appears indeed to " + "be '%s', but the types are different (check " + "that you are not e.g. mixing up different ffi " + "instances)", self.name, w_got.ctype.name) return oefmt(space.w_TypeError, "initializer for ctype '%s' must be a %s, not cdata " "'%s'", self.name, expected, w_got.ctype.name) diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -63,7 +63,7 @@ value = self._cast_result(value) else: value = self._cast_generic(w_ob) - w_cdata = cdataobj.W_CDataMem(space, self.size, self) + w_cdata = cdataobj.W_CDataMem(space, self) self.write_raw_integer_data(w_cdata, value) return w_cdata @@ -353,7 +353,7 @@ value = self.cast_unicode(w_ob) else: value = space.float_w(w_ob) - w_cdata = cdataobj.W_CDataMem(space, self.size, self) + w_cdata = cdataobj.W_CDataMem(space, self) if not isinstance(self, W_CTypePrimitiveLongDouble): w_cdata.write_raw_float_data(value) else: @@ -446,7 +446,7 @@ return self.space.wrap(value) def convert_to_object(self, cdata): - w_cdata = cdataobj.W_CDataMem(self.space, self.size, self) + w_cdata = cdataobj.W_CDataMem(self.space, self) with w_cdata as ptr: self._copy_longdouble(cdata, ptr) return w_cdata diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -185,7 +185,7 @@ self.is_void_ptr = isinstance(ctitem, ctypevoid.W_CTypeVoid) W_CTypePtrBase.__init__(self, space, size, extra, 2, ctitem) - def newp(self, w_init): + def newp(self, w_init, allocator): from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion space = self.space ctitem = self.ctitem @@ -205,14 +205,14 @@ datasize = ctitem.convert_struct_from_object( lltype.nullptr(rffi.CCHARP.TO), w_init, datasize) # - cdatastruct = cdataobj.W_CDataNewOwning(space, datasize, ctitem) + cdatastruct = allocator.allocate(space, datasize, ctitem) ptr = cdatastruct.unsafe_escaping_ptr() cdata = cdataobj.W_CDataPtrToStructOrUnion(space, ptr, self, cdatastruct) else: if self.is_char_or_unichar_ptr_or_array(): datasize *= 2 # forcefully add a null character - cdata = cdataobj.W_CDataNewOwning(space, datasize, self) + cdata = allocator.allocate(space, datasize, self) # if not space.is_w(w_init, space.w_None): with cdata as ptr: diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -81,10 +81,9 @@ def copy_and_convert_to_object(self, source): space = self.space self.check_complete() - ob = cdataobj.W_CDataNewOwning(space, self.size, self) - with ob as target: - misc._raw_memcopy(source, target, self.size) - return ob + ptr = lltype.malloc(rffi.CCHARP.TO, self.size, flavor='raw', zero=False) + misc._raw_memcopy(source, ptr, self.size) + return cdataobj.W_CDataNewStd(space, ptr, self) def typeoffsetof_field(self, fieldname, following): self.force_lazy_struct() diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -10,8 +10,8 @@ from pypy.module._cffi_backend import parse_c_type, realize_c_type from pypy.module._cffi_backend import newtype, cerrno, ccallback, ctypearray from pypy.module._cffi_backend import ctypestruct, ctypeptr, handle -from pypy.module._cffi_backend import cbuffer, func, cgc, wrapper -from pypy.module._cffi_backend import cffi_opcode +from pypy.module._cffi_backend import cbuffer, func, wrapper +from pypy.module._cffi_backend import cffi_opcode, allocator from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend.cdataobj import W_CData @@ -44,6 +44,10 @@ class W_FFIObject(W_Root): + ACCEPT_STRING = ACCEPT_STRING + ACCEPT_CTYPE = ACCEPT_CTYPE + ACCEPT_CDATA = ACCEPT_CDATA + w_gc_wref_remove = None @jit.dont_look_inside @@ -276,8 +280,9 @@ @unwrap_spec(w_python_callable=WrappedDefault(None), - w_error=WrappedDefault(None)) - def descr_callback(self, w_cdecl, w_python_callable, w_error): + w_error=WrappedDefault(None), + w_onerror=WrappedDefault(None)) + def descr_callback(self, w_cdecl, w_python_callable, w_error, w_onerror): """\ Return a callback object or a decorator making such a callback object. 'cdecl' must name a C function pointer type. The callback invokes the @@ -290,14 +295,16 @@ space = self.space if not space.is_none(w_python_callable): return ccallback.W_CDataCallback(space, w_ctype, - w_python_callable, w_error) + w_python_callable, w_error, + w_onerror) else: # decorator mode: returns a single-argument function - return space.appexec([w_ctype, w_error], - """(ctype, error): + return space.appexec([w_ctype, w_error, w_onerror], + """(ctype, error, onerror): import _cffi_backend return lambda python_callable: ( - _cffi_backend.callback(ctype, python_callable, error))""") + _cffi_backend.callback(ctype, python_callable, + error, onerror))""") def descr_cast(self, w_arg, w_ob): @@ -341,10 +348,7 @@ Later, when this new cdata object is garbage-collected, 'destructor(old_cdata_object)' will be called.""" # - return cgc.gc_weakrefs_build(self, w_cdata, w_destructor) - - def descr___gc_wref_remove(self, w_ref): - return cgc.gc_wref_remove(self, w_ref) + return w_cdata.with_gc(w_destructor) @unwrap_spec(replace_with=str) @@ -411,7 +415,31 @@ pointer to the memory somewhere else, e.g. into another structure.""" # w_ctype = self.ffi_type(w_arg, ACCEPT_STRING | ACCEPT_CTYPE) - return w_ctype.newp(w_init) + return w_ctype.newp(w_init, allocator.default_allocator) + + + @unwrap_spec(w_alloc=WrappedDefault(None), + w_free=WrappedDefault(None), + should_clear_after_alloc=int) + def descr_new_allocator(self, w_alloc, w_free, + should_clear_after_alloc=1): + """\ +Return a new allocator, i.e. a function that behaves like ffi.new() +but uses the provided low-level 'alloc' and 'free' functions. + +'alloc' is called with the size as argument. If it returns NULL, a +MemoryError is raised. 'free' is called with the result of 'alloc' +as argument. Both can be either Python function or directly C +functions. If 'free' is None, then no free function is called. +If both 'alloc' and 'free' are None, the default is used. + +If 'should_clear_after_alloc' is set to False, then the memory +returned by 'alloc' is assumed to be already cleared (or you are +fine with garbage); otherwise CFFI will clear it. + """ + # + return allocator.new_allocator(self, w_alloc, w_free, + should_clear_after_alloc) def descr_new_handle(self, w_arg): @@ -539,12 +567,17 @@ @jit.dont_look_inside -def W_FFIObject___new__(space, w_subtype, __args__): - r = space.allocate_instance(W_FFIObject, w_subtype) +def make_plain_ffi_object(space, w_ffitype=None): + if w_ffitype is None: + w_ffitype = space.gettypefor(W_FFIObject) + r = space.allocate_instance(W_FFIObject, w_ffitype) # get in 'src_ctx' a NULL which translation doesn't consider to be constant src_ctx = rffi.cast(parse_c_type.PCTX, 0) r.__init__(space, src_ctx) - return space.wrap(r) + return r + +def W_FFIObject___new__(space, w_subtype, __args__): + return space.wrap(make_plain_ffi_object(space, w_subtype)) def make_CData(space): return space.gettypefor(W_CData) @@ -578,7 +611,6 @@ W_FFIObject.set_errno, doc=W_FFIObject.doc_errno, cls=W_FFIObject), - __gc_wref_remove = interp2app(W_FFIObject.descr___gc_wref_remove), addressof = interp2app(W_FFIObject.descr_addressof), alignof = interp2app(W_FFIObject.descr_alignof), buffer = interp2app(W_FFIObject.descr_buffer), @@ -592,6 +624,7 @@ getctype = interp2app(W_FFIObject.descr_getctype), integer_const = interp2app(W_FFIObject.descr_integer_const), new = interp2app(W_FFIObject.descr_new), + new_allocator = interp2app(W_FFIObject.descr_new_allocator), new_handle = interp2app(W_FFIObject.descr_new_handle), offsetof = interp2app(W_FFIObject.descr_offsetof), sizeof = interp2app(W_FFIObject.descr_sizeof), diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -1,13 +1,13 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec, WrappedDefault -from pypy.module._cffi_backend import ctypeobj, cdataobj +from pypy.module._cffi_backend import ctypeobj, cdataobj, allocator # ____________________________________________________________ @unwrap_spec(w_ctype=ctypeobj.W_CType, w_init=WrappedDefault(None)) def newp(space, w_ctype, w_init): - return w_ctype.newp(w_init) + return w_ctype.newp(w_init, allocator.default_allocator) # ____________________________________________________________ @@ -18,9 +18,9 @@ # ____________________________________________________________ @unwrap_spec(w_ctype=ctypeobj.W_CType) -def callback(space, w_ctype, w_callable, w_error=None): +def callback(space, w_ctype, w_callable, w_error=None, w_onerror=None): from pypy.module._cffi_backend.ccallback import W_CDataCallback - return W_CDataCallback(space, w_ctype, w_callable, w_error) + return W_CDataCallback(space, w_ctype, w_callable, w_error, w_onerror) # ____________________________________________________________ @@ -105,3 +105,9 @@ "raw address on PyPy", w_x) # return cdataobj.W_CDataFromBuffer(space, _cdata, w_ctype, buf, w_x) + +# ____________________________________________________________ + + at unwrap_spec(w_cdata=cdataobj.W_CData) +def gcp(space, w_cdata, w_destructor): + return w_cdata.with_gc(w_destructor) diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py --- a/pypy/module/_cffi_backend/lib_obj.py +++ b/pypy/module/_cffi_backend/lib_obj.py @@ -102,6 +102,8 @@ # elif op == cffi_opcode.OP_GLOBAL_VAR: # A global variable of the exact type specified here + # (nowadays, only used by the ABI mode or backend + # compatibility; see OP_GLOBAL_F for the API mode w_ct = realize_c_type.realize_c_type( self.ffi, self.ctx.c_types, getarg(g.c_type_op)) g_size = rffi.cast(lltype.Signed, g.c_size_or_direct_fn) @@ -113,7 +115,13 @@ ptr = rffi.cast(rffi.CCHARP, g.c_address) if not ptr: # for dlopen() style ptr = self.cdlopen_fetch(attr) - w_result = cglob.W_GlobSupport(space, w_ct, ptr) + w_result = cglob.W_GlobSupport(space, attr, w_ct, ptr=ptr) + # + elif op == cffi_opcode.OP_GLOBAL_VAR_F: + w_ct = realize_c_type.realize_c_type( + self.ffi, self.ctx.c_types, getarg(g.c_type_op)) + w_result = cglob.W_GlobSupport(space, attr, w_ct, + fetch_addr=g.c_address) # elif (op == cffi_opcode.OP_CONSTANT_INT or op == cffi_opcode.OP_ENUM): @@ -131,6 +139,9 @@ realize_c_type.FUNCPTR_FETCH_CHARP, g.c_address) if w_ct.size <= 0: + raise oefmt(self.ffi.w_FFIError, + "constant '%s' is of type '%s', " + "whose size is not known", attr, w_ct.name) raise oefmt(space.w_SystemError, "constant has no known size") if not fetch_funcptr: # for dlopen() style @@ -172,7 +183,7 @@ w_value = self._build_attr(attr) if w_value is None: if is_getattr and attr == '__all__': - return self.dir1(ignore_type=cffi_opcode.OP_GLOBAL_VAR) + return self.dir1(ignore_global_vars=True) if is_getattr and attr == '__dict__': return self.full_dict_copy() if is_getattr and attr == '__name__': @@ -206,14 +217,18 @@ def descr_dir(self): return self.dir1() - def dir1(self, ignore_type=-1): + def dir1(self, ignore_global_vars=False): space = self.space total = rffi.getintfield(self.ctx, 'c_num_globals') g = self.ctx.c_globals names_w = [] for i in range(total): - if getop(g[i].c_type_op) != ignore_type: - names_w.append(space.wrap(rffi.charp2str(g[i].c_name))) + if ignore_global_vars: + op = getop(g[i].c_type_op) + if (op == cffi_opcode.OP_GLOBAL_VAR or + op == cffi_opcode.OP_GLOBAL_VAR_F): + continue + names_w.append(space.wrap(rffi.charp2str(g[i].c_name))) return space.newlist(names_w) def full_dict_copy(self): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1170,6 +1170,14 @@ BShort = new_primitive_type("short") BFunc = new_function_type((BShort,), BShort, False) f = callback(BFunc, Zcb1, -42) + # + seen = [] + oops_result = None + def oops(*args): + seen.append(args) + return oops_result + ff = callback(BFunc, Zcb1, -42, oops) + # orig_stderr = sys.stderr orig_getline = linecache.getline try: @@ -1195,6 +1203,59 @@ Trying to convert the result back to C: OverflowError: integer 60000 does not fit 'short' """) + sys.stderr = cStringIO.StringIO() + bigvalue = 20000 + assert len(seen) == 0 + assert ff(bigvalue) == -42 + assert sys.stderr.getvalue() == "" + assert len(seen) == 1 + exc, val, tb = seen[0] + assert exc is OverflowError + assert str(val) == "integer 60000 does not fit 'short'" + # + sys.stderr = cStringIO.StringIO() + bigvalue = 20000 + del seen[:] + oops_result = 81 + assert ff(bigvalue) == 81 + oops_result = None + assert sys.stderr.getvalue() == "" + assert len(seen) == 1 + exc, val, tb = seen[0] + assert exc is OverflowError + assert str(val) == "integer 60000 does not fit 'short'" + # + sys.stderr = cStringIO.StringIO() + bigvalue = 20000 + del seen[:] + oops_result = "xy" # not None and not an int! + assert ff(bigvalue) == -42 + oops_result = None + assert matches(sys.stderr.getvalue(), """\ +From cffi callback : +Trying to convert the result back to C: +OverflowError: integer 60000 does not fit 'short' + +During the call to 'onerror', another exception occurred: + +TypeError: $integer$ +""") + # + sys.stderr = cStringIO.StringIO() + seen = "not a list" # this makes the oops() function crash + assert ff(bigvalue) == -42 + assert matches(sys.stderr.getvalue(), """\ +From cffi callback : +Trying to convert the result back to C: +OverflowError: integer 60000 does not fit 'short' + +During the call to 'onerror', another exception occurred: + +Traceback (most recent call last): + File "$", line $, in oops + $ +AttributeError: 'str' object has no attribute 'append' +""") finally: sys.stderr = orig_stderr linecache.getline = orig_getline @@ -3341,6 +3402,29 @@ py.test.raises(RuntimeError, "p[42]") py.test.raises(RuntimeError, "p[42] = -1") +def test_mixup(): + BStruct1 = new_struct_type("foo") + BStruct2 = new_struct_type("foo") # <= same name as BStruct1 + BStruct3 = new_struct_type("bar") + BStruct1Ptr = new_pointer_type(BStruct1) + BStruct2Ptr = new_pointer_type(BStruct2) + BStruct3Ptr = new_pointer_type(BStruct3) + BStruct1PtrPtr = new_pointer_type(BStruct1Ptr) + BStruct2PtrPtr = new_pointer_type(BStruct2Ptr) + BStruct3PtrPtr = new_pointer_type(BStruct3Ptr) + pp1 = newp(BStruct1PtrPtr) + pp2 = newp(BStruct2PtrPtr) + pp3 = newp(BStruct3PtrPtr) + pp1[0] = pp1[0] + e = py.test.raises(TypeError, "pp3[0] = pp1[0]") + assert str(e.value).startswith("initializer for ctype 'bar *' must be a ") + assert str(e.value).endswith(", not cdata 'foo *'") + e = py.test.raises(TypeError, "pp2[0] = pp1[0]") + assert str(e.value) == ("initializer for ctype 'foo *' appears indeed to " + "be 'foo *', but the types are different (check " + "that you are not e.g. mixing up different ffi " + "instances)") + def test_version(): # this test is here mostly for PyPy - assert __version__ == "1.1.2" + assert __version__ == "1.2.0" diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py --- a/pypy/module/_cffi_backend/test/test_ffi_obj.py +++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py @@ -114,6 +114,18 @@ assert ffi.callback("int(int)", lambda x: x + "", -66)(10) == -66 assert ffi.callback("int(int)", lambda x: x + "", error=-66)(10) == -66 + def test_ffi_callback_onerror(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + seen = [] + def myerror(exc, val, tb): + seen.append(exc) + cb = ffi.callback("int(int)", lambda x: x + "", onerror=myerror) + assert cb(10) == 0 + cb = ffi.callback("int(int)", lambda x:int(1E100), -66, onerror=myerror) + assert cb(10) == -66 + assert seen == [TypeError, OverflowError] + def test_ffi_callback_decorator(self): import _cffi_backend as _cffi1_backend ffi = _cffi1_backend.FFI() @@ -122,6 +134,37 @@ assert deco(lambda x: x + "")(10) == -66 assert deco(lambda x: x + 42)(10) == 52 + def test_ffi_callback_onerror(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + seen = [] + def oops(*args): + seen.append(args) + + @ffi.callback("int(int)", onerror=oops) + def fn1(x): + return x + "" + assert fn1(10) == 0 + + @ffi.callback("int(int)", onerror=oops, error=-66) + def fn2(x): + return x + "" + assert fn2(10) == -66 + + assert len(seen) == 2 + exc, val, tb = seen[0] + assert exc is TypeError + assert isinstance(val, TypeError) + assert tb.tb_frame.f_code.co_name == "fn1" + exc, val, tb = seen[1] + assert exc is TypeError + assert isinstance(val, TypeError) + assert tb.tb_frame.f_code.co_name == "fn2" + del seen[:] + # + raises(TypeError, ffi.callback, "int(int)", + lambda x: x, onerror=42) # <- not callable + def test_ffi_getctype(self): import _cffi_backend as _cffi1_backend ffi = _cffi1_backend.FFI() @@ -228,3 +271,99 @@ import gc gc.collect() assert seen == [1] + + def test_ffi_new_allocator_1(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + alloc1 = ffi.new_allocator() + alloc2 = ffi.new_allocator(should_clear_after_alloc=False) + for retry in range(100): + p1 = alloc1("int[10]") + p2 = alloc2("int[10]") + combination = 0 + for i in range(10): + assert p1[i] == 0 + combination |= p2[i] + p1[i] = -42 + p2[i] = -43 + if combination != 0: + break + del p1, p2 + import gc; gc.collect() + else: + raise AssertionError("cannot seem to get an int[10] not " + "completely cleared") + + def test_ffi_new_allocator_2(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + seen = [] + def myalloc(size): + seen.append(size) + return ffi.new("char[]", "X" * size) + def myfree(raw): + seen.append(raw) + alloc1 = ffi.new_allocator(myalloc, myfree) + alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree, + should_clear_after_alloc=False) + p1 = alloc1("int[10]") + p2 = alloc2("int[]", 10) + assert seen == [40, 40] + assert ffi.typeof(p1) == ffi.typeof("int[10]") + assert ffi.sizeof(p1) == 40 + assert ffi.typeof(p2) == ffi.typeof("int[]") + assert ffi.sizeof(p2) == 40 + assert p1[5] == 0 + assert p2[6] == ord('X') * 0x01010101 + raw1 = ffi.cast("char *", p1) + raw2 = ffi.cast("char *", p2) + del p1, p2 + retries = 0 + while len(seen) != 4: + retries += 1 + assert retries <= 5 + import gc; gc.collect() + assert seen == [40, 40, raw1, raw2] + assert repr(seen[2]) == "" + assert repr(seen[3]) == "" + + def test_ffi_new_allocator_3(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + seen = [] + def myalloc(size): + seen.append(size) + return ffi.new("char[]", "X" * size) + alloc1 = ffi.new_allocator(myalloc) # no 'free' + p1 = alloc1("int[10]") + assert seen == [40] + assert ffi.typeof(p1) == ffi.typeof("int[10]") + assert ffi.sizeof(p1) == 40 + assert p1[5] == 0 + + def test_ffi_new_allocator_4(self): + import _cffi_backend as _cffi1_backend + ffi = _cffi1_backend.FFI() + raises(TypeError, ffi.new_allocator, free=lambda x: None) + # + def myalloc2(size): + raise LookupError + alloc2 = ffi.new_allocator(myalloc2) + raises(LookupError, alloc2, "int[5]") + # + def myalloc3(size): + return 42 + alloc3 = ffi.new_allocator(myalloc3) From noreply at buildbot.pypy.org Sun Aug 2 15:13:11 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 15:13:11 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: in-progress Message-ID: <20150802131311.26DDE1C0352@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78738:e34fc115c19d Date: 2015-08-02 15:13 +0200 http://bitbucket.org/pypy/pypy/changeset/e34fc115c19d/ Log: in-progress diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/__init__.py @@ -0,0 +1,2 @@ +from rpython.rlib.rvmprof.rvmprof import get_vmprof +from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/cintf.py @@ -0,0 +1,5 @@ + + +def vmprof_init(): pass +def vmprof_enable(fileno, interval_usec): return 0 +def vmprof_ignore_signals(ignore): pass diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -1,13 +1,21 @@ -import sys -from rpython.rlib.objectmodel import specialize +import sys, os +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rstring import StringBuilder -from rpython.rlib import rgc +from rpython.rlib import jit, rgc +from rpython.rlib.rvmprof import cintf MAX_CODES = 8000 # ____________________________________________________________ +class VMProfError(Exception): + def __init__(self, msg): + self.msg = msg + def __str__(self): + return self.msg + + class VMProf(object): def __init__(self): @@ -17,6 +25,9 @@ self._cleanup_() def _cleanup_(self): + self.is_enabled = False + self.ever_enabled = False + self.fileno = -1 self._current_codes = None if sys.maxint == 2147483647: self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit @@ -27,21 +38,12 @@ def register_code(self, code, name): """Register the code object. Call when a new code object is made. """ - if code._vmprof_unique_id != 0: - return - uid = self._code_unique_id + 4 # so we have two bits to mark stuff - code._vmprof_unique_id = uid - self._code_unique_id = uid - # - b = self._current_codes - if b is None: - b = self._current_codes = StringBuilder() - b.append('\x02') - write_long_to_string_builder(uid, b) - write_long_to_string_builder(len(name), b) - b.append(name) - if b.getlength() >= MAX_CODES: - self._flush_codes() + if code._vmprof_unique_id == 0: + uid = self._code_unique_id + 1 + code._vmprof_unique_id = uid + self._code_unique_id = uid + if self.is_enabled: + self._write_code_registration(uid, name) def register_code_object_class(self, CodeClass, full_name_func): """NOT_RPYTHON @@ -61,21 +63,93 @@ CodeClass._vmprof_unique_id = 0 # default value: "unknown" self._code_classes.add(CodeClass) # - def try_cast_to_pycode(gcref): + def try_cast_to_code(gcref): return rgc.try_cast_gcref_to_instance(CodeClass, gcref) # def gather_all_code_objs(): - all_code_objs = rgc.do_get_objects(try_cast_to_pycode) + all_code_objs = rgc.do_get_objects(try_cast_to_code) for code in all_code_objs: - self.register_code(code, full_name_func(code)) + uid = code._vmprof_unique_id + if uid != 0: + self._write_code_registration(uid, full_name_func(code)) prev() # make a chained list of the gather() functions for all # the types of code objects prev = self._gather_all_code_objs self._gather_all_code_objs = gather_all_code_objs + def enable(self, fileno, interval): + """Enable vmprof. Writes go to the given 'fileno'. + The sampling interval is given by 'interval' as a number of + seconds, as a float which must be not greater than 1.0. + Raises VMProfError if something goes wrong. + """ + assert fileno >= 0 + if self.is_enabled: + raise VMProfError("vmprof is already enabled") + if not (1e-6 <= interval <= 1.0): + raise VMProfError("bad value for 'interval'") + interval_usec = int(interval * 1000000.0) + # + self.fileno = fileno + self.is_enabled = True + self._write_header(interval_usec) + if not self.ever_enabled: + if we_are_translated(): + p_error = cintf.vmprof_init() + if p_error: + raise VMProfError(rffi.charp2str(p_error)) + self.ever_enabled = True + self._gather_all_code_objs() + if we_are_translated(): + # does not work untranslated + res = cintf.vmprof_enable(fileno, interval_usec) + if res < 0: + raise VMProfError(os.strerror(rposix.get_saved_errno())) -def write_long_to_string_builder(l, b): + def _write_code_registration(self, uid, name): + b = self._current_codes + if b is None: + b = self._current_codes = StringBuilder() + b.append('\x02') + _write_long_to_string_builder(uid, b) + _write_long_to_string_builder(len(name), b) + b.append(name) + if b.getlength() >= MAX_CODES: + self._flush_codes() + + def _flush_codes(self): + buf = self._current_codes.build() + self._current_codes = None + self._carefully_write(buf) + + def _carefully_write(self, buf): + fd = self._fileno + assert fd >= 0 + if not buf: + return + cintf.vmprof_ignore_signals(True) + try: + while len(buf) > 0: + num = os.write(fd, buf) + buf = buf[num:] + finally: + cintf.vmprof_ignore_signals(False) + + def _write_header(self, interval_usec): + b = StringBuilder() + _write_long_to_string_builder(0, b) + _write_long_to_string_builder(3, b) + _write_long_to_string_builder(0, b) + _write_long_to_string_builder(interval_usec, b) + _write_long_to_string_builder(0, b) + b.append('\x04') # interp name + b.append(chr(len('pypy'))) + b.append('pypy') + self._carefully_write(b.build()) + + +def _write_long_to_string_builder(l, b): b.append(chr(l & 0xff)) b.append(chr((l >> 8) & 0xff)) b.append(chr((l >> 16) & 0xff)) @@ -87,6 +161,68 @@ b.append(chr((l >> 56) & 0xff)) +def vmprof_execute_code(get_code_fn, result_class=None): + """Decorator to be used on the function that interprets a code object. + + get_code_fn(*args) is called to extract the code object from the + arguments given to the decorated function. + + The original function can return None, an integer, or an instance. + In the latter case (only), 'result_class' must be set. + + NOTE: for now, this assumes that the decorated functions only takes + instances or plain integer arguments, and at most 5 of them + (including 'self' if applicable). + """ + def lower(*args): + if len(args) == 0: + return (), "" + ll_args, token = lower(*args[1:]) + ll_arg = args[0] + if isintance(ll_arg, int): + tok = "i" + else: + tok = "r" + ll_arg = cast_instance_to_gcref(ll_arg) + return (ll_arg,) + ll_args, tok + token + + @specialize.memo() + def get_ll_trampoline(func, token): + xxx + + def decorate(func): + def decorated_function(*args): + # go through the asm trampoline ONLY if we are translated but not + # being JITted. + # + # If we are not translated, we obviously don't want to go through + # the trampoline because there is no C function it can call. + # + # If we are being JITted, we want to skip the trampoline, else the + # JIT cannot see through it. + # + if we_are_translated() and not jit.we_are_jitted(): + # if we are translated, call the trampoline + unique_id = get_code_fn(*args)._vmprof_unique_id + ll_args, token = lower(*args) + ll_trampoline = get_ll_trampoline(func, token) + ll_result = ll_trampoline(*ll_args + (unique_id,)) + if result_class is not None: + return cast_base_ptr_to_instance(result_class, ll_result) + else: + return ll_result + else: + return func(*args) + decorated_function.__name__ = func.__name__ + '_rvmprof' + return decorated_function + return decorate + + +_vmprof_instance = None + @specialize.memo() def get_vmprof(): - return VMProf() + global _vmprof_instance + if _vmprof_instance is None: + _vmprof_instance = VMProf() + return _vmprof_instance diff --git a/rpython/rlib/rvmprof/test/__init__.py b/rpython/rlib/rvmprof/test/__init__.py new file mode 100644 diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -0,0 +1,14 @@ +from rpython.rlib.rvmprof import get_vmprof, vmprof_execute_code + + +def test_vmprof_execute_code_1(): + + class MyCode: + pass + get_vmprof().register_code_object_class(MyCode, lambda code: 'some code') + + @vmprof_execute_code(lambda code, num: code) + def main(code, num): + print num + + main(MyCode(), 5) From noreply at buildbot.pypy.org Sun Aug 2 15:56:49 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 15:56:49 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Pass the translation tests for the generated trampolines Message-ID: <20150802135649.A13F01C1185@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78739:251e3d7a6609 Date: 2015-08-02 15:57 +0200 http://bitbucket.org/pypy/pypy/changeset/251e3d7a6609/ Log: Pass the translation tests for the generated trampolines diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -1,5 +1,83 @@ +from rpython.tool.udir import udir +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo def vmprof_init(): pass def vmprof_enable(fileno, interval_usec): return 0 def vmprof_ignore_signals(ignore): pass + + +def token2lltype(tok): + if tok == 'i': + return lltype.Signed + if tok == 'r': + return llmemory.GCREF + raise NotImplementedError(repr(tok)) + +def make_trampoline_function(name, func, token, restok): + from rpython.jit.backend import detect_cpu + + cont_name = 'vmprof_f_%s_%s' % (name, token) + tramp_name = 'vmprof_t_%s_%s' % (name, token) + + func.c_name = cont_name + func._dont_inline_ = True + + assert detect_cpu.autodetect() == detect_cpu.MODEL_X86_64, ( + "rvmprof only supports x86-64 CPUs for now") + + # mapping of argument count (not counting the final uid argument) to + # the register that holds this uid argument + reg = {0: '%rdi', + 1: '%rsi', + 2: '%rdx', + 3: '%rcx', + 4: '%r8', + 5: '%r9', + } + reg = reg[len(token)] + + target = udir.join('module_cache') + target.ensure(dir=1) + target = target.join('trampoline_%s_%s.vmprof.s' % (name, token)) + target.write("""\ +\t.text +\t.globl\t%(tramp_name)s +\t.type\t%(tramp_name)s, @function +%(tramp_name)s: +\t.cfi_startproc +\tpushq\t%(reg)s +\t.cfi_def_cfa_offset 16 +\tcall %(cont_name)s at PLT +\taddq\t$8, %%rsp +\t.cfi_def_cfa_offset 8 +\tret +\t.cfi_endproc +\t.size\t%(tramp_name)s, .-%(tramp_name)s +""" % locals()) + + def tok2cname(tok): + if tok == 'i': + return 'long' + if tok == 'r': + return 'void *' + raise NotImplementedError(repr(tok)) + + header = '%s %s(%s);\n' % ( + tok2cname(restok), + tramp_name, + ', '.join([tok2cname(tok) for tok in token] + ['long'])) + + eci = ExternalCompilationInfo( + post_include_bits = [header], + separate_module_files = [str(target)], + ) + + return rffi.llexternal( + tramp_name, + [token2lltype(tok) for tok in token] + [lltype.Signed], + token2lltype(restok), + compilation_info=eci, + _nowrapper=True, sandboxsafe=True, + random_effects_on_gcobjs=True) diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -3,6 +3,8 @@ from rpython.rlib.rstring import StringBuilder from rpython.rlib import jit, rgc from rpython.rlib.rvmprof import cintf +from rpython.rtyper.annlowlevel import cast_instance_to_gcref +from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance MAX_CODES = 8000 @@ -161,10 +163,12 @@ b.append(chr((l >> 56) & 0xff)) -def vmprof_execute_code(get_code_fn, result_class=None): +def vmprof_execute_code(name, get_code_fn, result_class=None): """Decorator to be used on the function that interprets a code object. - get_code_fn(*args) is called to extract the code object from the + 'name' must be a unique name. + + 'get_code_fn(*args)' is called to extract the code object from the arguments given to the decorated function. The original function can return None, an integer, or an instance. @@ -174,23 +178,30 @@ instances or plain integer arguments, and at most 5 of them (including 'self' if applicable). """ - def lower(*args): - if len(args) == 0: - return (), "" - ll_args, token = lower(*args[1:]) - ll_arg = args[0] - if isintance(ll_arg, int): - tok = "i" - else: - tok = "r" - ll_arg = cast_instance_to_gcref(ll_arg) - return (ll_arg,) + ll_args, tok + token + def decorate(func): - @specialize.memo() - def get_ll_trampoline(func, token): - xxx + def lower(*args): + if len(args) == 0: + return (), "" + ll_args, token = lower(*args[1:]) + ll_arg = args[0] + if isinstance(ll_arg, int): + tok = "i" + else: + tok = "r" + ll_arg = cast_instance_to_gcref(ll_arg) + return (ll_arg,) + ll_args, tok + token - def decorate(func): + @specialize.memo() + def get_ll_trampoline(token): + assert len(token) <= 5, ( + "not supported: %r takes more than 5 arguments" % (func,)) + if result_class is None: + restok = "i" + else: + restok = "r" + return cintf.make_trampoline_function(name, func, token, restok) + def decorated_function(*args): # go through the asm trampoline ONLY if we are translated but not # being JITted. @@ -205,7 +216,7 @@ # if we are translated, call the trampoline unique_id = get_code_fn(*args)._vmprof_unique_id ll_args, token = lower(*args) - ll_trampoline = get_ll_trampoline(func, token) + ll_trampoline = get_ll_trampoline(token) ll_result = ll_trampoline(*ll_args + (unique_id,)) if result_class is not None: return cast_base_ptr_to_instance(result_class, ll_result) @@ -215,6 +226,7 @@ return func(*args) decorated_function.__name__ = func.__name__ + '_rvmprof' return decorated_function + return decorate diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -1,4 +1,5 @@ from rpython.rlib.rvmprof import get_vmprof, vmprof_execute_code +from rpython.translator.c.test.test_genc import compile def test_vmprof_execute_code_1(): @@ -7,8 +8,40 @@ pass get_vmprof().register_code_object_class(MyCode, lambda code: 'some code') - @vmprof_execute_code(lambda code, num: code) + @vmprof_execute_code("xcode1", lambda code, num: code) def main(code, num): print num + return 42 - main(MyCode(), 5) + def f(): + res = main(MyCode(), 5) + assert res == 42 + return 0 + + assert f() == 0 + fn = compile(f, []) + assert fn() == 0 + + +def test_vmprof_execute_code_2(): + + class MyCode: + pass + get_vmprof().register_code_object_class(MyCode, lambda code: 'some code') + + class A: + pass + + @vmprof_execute_code("xcode2", lambda num, code: code, result_class=A) + def main(num, code): + print num + return A() + + def f(): + a = main(7, MyCode()) + assert isinstance(a, A) + return 0 + + assert f() == 0 + fn = compile(f, []) + assert fn() == 0 From noreply at buildbot.pypy.org Sun Aug 2 15:58:14 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 15:58:14 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Skip running the tests based on platform Message-ID: <20150802135814.C319E1C1185@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78740:3776642bc6fc Date: 2015-08-02 15:58 +0200 http://bitbucket.org/pypy/pypy/changeset/3776642bc6fc/ Log: Skip running the tests based on platform diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -1,5 +1,10 @@ +import py from rpython.rlib.rvmprof import get_vmprof, vmprof_execute_code from rpython.translator.c.test.test_genc import compile +from rpython.jit.backend import detect_cpu + +if detect_cpu.autodetect() != detect_cpu.MODEL_X86_64: + py.test.skip("rvmprof only supports x86-64 CPUs for now") def test_vmprof_execute_code_1(): From noreply at buildbot.pypy.org Sun Aug 2 16:27:06 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 16:27:06 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: in-progress Message-ID: <20150802142706.CB6C21C1185@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78741:fcf2c4f1208a Date: 2015-08-02 16:27 +0200 http://bitbucket.org/pypy/pypy/changeset/fcf2c4f1208a/ Log: in-progress diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -1,6 +1,46 @@ +import py +import sys from rpython.tool.udir import udir from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.tool import rffi_platform as platform + + +ROOT = py.path.local(__file__).join('..') +SRC = ROOT.join('src') + + +if sys.platform.startswith('linux'): + libs = ['dl'] +else: + libs = [] + +eci_kwds = dict( + include_dirs = [SRC], + includes = ['rvmprof.h'], + libraries = libs, + separate_module_files = [SRC.join('rvmprof.c')], + ) +eci = ExternalCompilationInfo(**eci_kwds) + +platform.verify_eci(eci) + + +vmprof_init = rffi.llexternal("rpython_vmprof_init", [], rffi.CCHARP, + compilation_info=eci) +## vmprof_enable = rffi.llexternal("vmprof_enable", +## [rffi.INT, rffi.LONG, rffi.INT, +## rffi.CCHARP, rffi.INT], +## rffi.INT, compilation_info=eci, +## save_err=rffi.RFFI_SAVE_ERRNO) +## vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, +## compilation_info=eci, +## save_err=rffi.RFFI_SAVE_ERRNO) + +## vmprof_register_virtual_function = rffi.llexternal( +## "vmprof_register_virtual_function", +## [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, +## compilation_info=eci, _nowrapper=True) def vmprof_init(): pass @@ -36,7 +76,11 @@ 4: '%r8', 5: '%r9', } - reg = reg[len(token)] + try: + reg = reg[len(token)] + except KeyError: + raise NotImplementedError( + "not supported: %r takes more than 5 arguments" % (func,)) target = udir.join('module_cache') target.ensure(dir=1) @@ -69,6 +113,19 @@ tramp_name, ', '.join([tok2cname(tok) for tok in token] + ['long'])) + header += """\ +static int cmp_%s(void *addr) { + if (addr == %s) return 1; +#ifdef VMPROF_ADDR_OF_TRAMPOLINE + return VMPROF_ADDR_OF_TRAMPOLINE(addr); +#undef VMPROF_ADDR_OF_TRAMPOLINE +#else + return 0; +#endif +#define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s +} +""" % (tramp_name, tramp_name, tramp_name) + eci = ExternalCompilationInfo( post_include_bits = [header], separate_module_files = [str(target)], diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -194,8 +194,6 @@ @specialize.memo() def get_ll_trampoline(token): - assert len(token) <= 5, ( - "not supported: %r takes more than 5 arguments" % (func,)) if result_class is None: restok = "i" else: diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -0,0 +1,11 @@ + + +#ifndef VMPROF_ADDR_OF_TRAMPOLINE +# error "RPython program using rvmprof, but not calling vmprof_execute_code()" +#endif + + +char *rpython_vmprof_init(void) +{ + xxxx; +} diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -0,0 +1,2 @@ + +char *rpython_vmprof_init(void); diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -50,3 +50,26 @@ assert f() == 0 fn = compile(f, []) assert fn() == 0 + + +def test_register_code(): + + class MyCode: + pass + get_vmprof().register_code_object_class(MyCode, lambda code: 'some code') + + @vmprof_execute_code("xcode1", lambda code, num: code) + def main(code, num): + print num + return 42 + + def f(): + code = MyCode() + get_vmprof().register_code(code, 'some code') + res = main(code, 5) + assert res == 42 + return 0 + + assert f() == 0 + fn = compile(f, []) + assert fn() == 0 From noreply at buildbot.pypy.org Sun Aug 2 16:35:09 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 16:35:09 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Test and fixes for enable() Message-ID: <20150802143509.19FAB1C1247@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78742:1d75a1bd5d77 Date: 2015-08-02 16:35 +0200 http://bitbucket.org/pypy/pypy/changeset/1d75a1bd5d77/ Log: Test and fixes for enable() diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -1,10 +1,11 @@ import sys, os from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rstring import StringBuilder -from rpython.rlib import jit, rgc +from rpython.rlib import jit, rgc, rposix from rpython.rlib.rvmprof import cintf from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance +from rpython.rtyper.lltypesystem import rffi MAX_CODES = 8000 @@ -126,7 +127,7 @@ self._carefully_write(buf) def _carefully_write(self, buf): - fd = self._fileno + fd = self.fileno assert fd >= 0 if not buf: return diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -1,4 +1,5 @@ -import py +import py, os +from rpython.tool.udir import udir from rpython.rlib.rvmprof import get_vmprof, vmprof_execute_code from rpython.translator.c.test.test_genc import compile from rpython.jit.backend import detect_cpu @@ -73,3 +74,33 @@ assert f() == 0 fn = compile(f, []) assert fn() == 0 + + +def test_enable(): + + class MyCode: + pass + get_vmprof().register_code_object_class(MyCode, lambda code: 'some code') + + @vmprof_execute_code("xcode1", lambda code, num: code) + def main(code, num): + print num + return 42 + + tmpfilename = str(udir.join('test_rvmprof')) + + def f(): + code = MyCode() + get_vmprof().register_code(code, 'some code') + fd = os.open(tmpfilename, os.O_WRONLY | os.O_CREAT, 0666) + get_vmprof().enable(fd, 0.5) + res = main(code, 5) + assert res == 42 + return 0 + + assert f() == 0 + assert os.path.exists(tmpfilename) + fn = compile(f, [], gcpolicy="minimark") + os.unlink(tmpfilename) + assert fn() == 0 + assert os.path.exists(tmpfilename) From noreply at buildbot.pypy.org Sun Aug 2 16:49:36 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 16:49:36 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Add some files from pypy/module/_vmprof/src Message-ID: <20150802144936.1F8531C1247@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78743:0f949cb3ea81 Date: 2015-08-02 16:49 +0200 http://bitbucket.org/pypy/pypy/changeset/0f949cb3ea81/ Log: Add some files from pypy/module/_vmprof/src diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -43,7 +43,6 @@ ## compilation_info=eci, _nowrapper=True) -def vmprof_init(): pass def vmprof_enable(fileno, interval_usec): return 0 def vmprof_ignore_signals(ignore): pass diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -1,11 +1,46 @@ - +#define _GNU_SOURCE 1 +#include "common_header.h" #ifndef VMPROF_ADDR_OF_TRAMPOLINE # error "RPython program using rvmprof, but not calling vmprof_execute_code()" #endif +#include "rvmprof_getpc.h" +#include "rvmprof_base.h" +#include + + +/************************************************************/ + +// functions copied from libunwind using dlopen + +static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL; +static int (*unw_step)(unw_cursor_t*) = NULL; +static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; +static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; + + char *rpython_vmprof_init(void) { - xxxx; + if (!unw_get_reg) { + void *libhandle; + + if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) + goto error; + if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) + goto error; + if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))) + goto error; + if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) + goto error; + if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) + goto error; + } + return NULL; + + error: + return dlerror(); } + +/************************************************************/ diff --git a/rpython/rlib/rvmprof/src/rvmprof_base.h b/rpython/rlib/rvmprof/src/rvmprof_base.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/rvmprof_base.h @@ -0,0 +1,90 @@ +#include +#include +#include + +// copied from libunwind.h + +typedef enum + { + UNW_X86_64_RAX, + UNW_X86_64_RDX, + UNW_X86_64_RCX, + UNW_X86_64_RBX, + UNW_X86_64_RSI, + UNW_X86_64_RDI, + UNW_X86_64_RBP, + UNW_X86_64_RSP, + UNW_X86_64_R8, + UNW_X86_64_R9, + UNW_X86_64_R10, + UNW_X86_64_R11, + UNW_X86_64_R12, + UNW_X86_64_R13, + UNW_X86_64_R14, + UNW_X86_64_R15, + UNW_X86_64_RIP, +#ifdef CONFIG_MSABI_SUPPORT + UNW_X86_64_XMM0, + UNW_X86_64_XMM1, + UNW_X86_64_XMM2, + UNW_X86_64_XMM3, + UNW_X86_64_XMM4, + UNW_X86_64_XMM5, + UNW_X86_64_XMM6, + UNW_X86_64_XMM7, + UNW_X86_64_XMM8, + UNW_X86_64_XMM9, + UNW_X86_64_XMM10, + UNW_X86_64_XMM11, + UNW_X86_64_XMM12, + UNW_X86_64_XMM13, + UNW_X86_64_XMM14, + UNW_X86_64_XMM15, + UNW_TDEP_LAST_REG = UNW_X86_64_XMM15, +#else + UNW_TDEP_LAST_REG = UNW_X86_64_RIP, +#endif + + /* XXX Add other regs here */ + + /* frame info (read-only) */ + UNW_X86_64_CFA, + + UNW_TDEP_IP = UNW_X86_64_RIP, + UNW_TDEP_SP = UNW_X86_64_RSP, + UNW_TDEP_BP = UNW_X86_64_RBP, + UNW_TDEP_EH = UNW_X86_64_RAX + } +x86_64_regnum_t; + +typedef uint64_t unw_word_t; + +#define UNW_TDEP_CURSOR_LEN 127 + +typedef struct unw_cursor + { + unw_word_t opaque[UNW_TDEP_CURSOR_LEN]; + } +unw_cursor_t; + +#define UNW_REG_IP UNW_X86_64_RIP +#define UNW_REG_SP UNW_X86_64_RSP + +typedef ucontext_t unw_context_t; + +typedef struct unw_proc_info + { + unw_word_t start_ip; /* first IP covered by this procedure */ + unw_word_t end_ip; /* first IP NOT covered by this procedure */ + unw_word_t lsda; /* address of lang.-spec. data area (if any) */ + unw_word_t handler; /* optional personality routine */ + unw_word_t gp; /* global-pointer value for this procedure */ + unw_word_t flags; /* misc. flags */ + + int format; /* unwind-info format (arch-specific) */ + int unwind_info_size; /* size of the information (if applicable) */ + void *unwind_info; /* unwind-info (arch-specific) */ + } +unw_proc_info_t; + +// end of copy diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/rvmprof_config.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/rvmprof_config.h @@ -0,0 +1,6 @@ +#define HAVE_SYS_UCONTEXT_H +#if defined(__FreeBSD__) || defined(__APPLE__) +#define PC_FROM_UCONTEXT uc_mcontext.mc_rip +#else +#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP] +#endif diff --git a/pypy/module/_vmprof/src/getpc.h b/rpython/rlib/rvmprof/src/rvmprof_getpc.h rename from pypy/module/_vmprof/src/getpc.h rename to rpython/rlib/rvmprof/src/rvmprof_getpc.h --- a/pypy/module/_vmprof/src/getpc.h +++ b/rpython/rlib/rvmprof/src/rvmprof_getpc.h @@ -44,7 +44,7 @@ #ifndef BASE_GETPC_H_ #define BASE_GETPC_H_ -#include "config.h" +#include "rvmprof_config.h" // On many linux systems, we may need _GNU_SOURCE to get access to // the defined constants that define the register we want to see (eg From noreply at buildbot.pypy.org Sun Aug 2 17:12:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 17:12:31 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Tweaks tweaks until the tests pass Message-ID: <20150802151231.4EA7E1C0352@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78744:ff54f7fd27c2 Date: 2015-08-02 17:11 +0200 http://bitbucket.org/pypy/pypy/changeset/ff54f7fd27c2/ Log: Tweaks tweaks until the tests pass diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -42,9 +42,12 @@ ## [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, ## compilation_info=eci, _nowrapper=True) +vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals", + [rffi.INT], lltype.Void, + compilation_info=eci) + def vmprof_enable(fileno, interval_usec): return 0 -def vmprof_ignore_signals(ignore): pass def token2lltype(tok): diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -1,8 +1,18 @@ #define _GNU_SOURCE 1 -#include "common_header.h" -#ifndef VMPROF_ADDR_OF_TRAMPOLINE -# error "RPython program using rvmprof, but not calling vmprof_execute_code()" + +#if defined(RPY_EXTERN) && !defined(RPY_EXPORTED) + /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */ +# define RPY_EXPORTED extern __attribute__((visibility("default"))) + +#else + +# include "common_header.h" +# include "rvmprof.h" +# ifndef VMPROF_ADDR_OF_TRAMPOLINE +# error "RPython program using rvmprof, but not calling vmprof_execute_code()" +# endif + #endif @@ -21,6 +31,7 @@ static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; +RPY_EXTERN char *rpython_vmprof_init(void) { if (!unw_get_reg) { @@ -44,3 +55,18 @@ } /************************************************************/ + +static long volatile ignore_signals = 0; + +RPY_EXTERN +void rpython_vmprof_ignore_signals(int ignored) +{ +#ifndef _MSC_VER + if (ignored) + __sync_lock_test_and_set(&ignore_signals, 1); + else + __sync_lock_release(&ignore_signals); +#else + _InterlockedExchange(&ignore_signals, (long)ignored); +#endif +} diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,2 +1,3 @@ -char *rpython_vmprof_init(void); +RPY_EXTERN char *rpython_vmprof_init(void); +RPY_EXTERN void vmprof_ignore_signals(int); From noreply at buildbot.pypy.org Sun Aug 2 17:41:43 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 17:41:43 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Enable(), disable() Message-ID: <20150802154143.656371C02DE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78745:6ea3a10ad04e Date: 2015-08-02 17:41 +0200 http://bitbucket.org/pypy/pypy/changeset/6ea3a10ad04e/ Log: Enable(), disable() diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -11,7 +11,7 @@ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction, CodeUniqueIds) + UserDelAction) from pypy.interpreter.error import OperationError, new_exception_class, oefmt from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary @@ -391,7 +391,6 @@ self.actionflag = ActionFlag() # changed by the signal module self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) - self.code_unique_ids = CodeUniqueIds() self._code_of_sys_exc_info = None # can be overridden to a subclass @@ -670,16 +669,6 @@ assert ec is not None return ec - def register_code_callback(self, callback): - cui = self.code_unique_ids - cui.code_callback = callback - - def register_code_object(self, pycode): - cui = self.code_unique_ids - if cui.code_callback is None: - return - cui.code_callback(self, pycode) - def _freeze_(self): return True diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -28,14 +28,13 @@ vmprof_init = rffi.llexternal("rpython_vmprof_init", [], rffi.CCHARP, compilation_info=eci) -## vmprof_enable = rffi.llexternal("vmprof_enable", -## [rffi.INT, rffi.LONG, rffi.INT, -## rffi.CCHARP, rffi.INT], -## rffi.INT, compilation_info=eci, -## save_err=rffi.RFFI_SAVE_ERRNO) -## vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, -## compilation_info=eci, -## save_err=rffi.RFFI_SAVE_ERRNO) +vmprof_enable = rffi.llexternal("rpython_vmprof_enable", + [rffi.INT, rffi.LONG], + rffi.INT, compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) +vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT, + compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) ## vmprof_register_virtual_function = rffi.llexternal( ## "vmprof_register_virtual_function", @@ -47,9 +46,6 @@ compilation_info=eci) -def vmprof_enable(fileno, interval_usec): return 0 - - def token2lltype(tok): if tok == 'i': return lltype.Signed diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -84,18 +84,17 @@ def enable(self, fileno, interval): """Enable vmprof. Writes go to the given 'fileno'. The sampling interval is given by 'interval' as a number of - seconds, as a float which must be not greater than 1.0. + seconds, as a float which must be smaller than 1.0. Raises VMProfError if something goes wrong. """ assert fileno >= 0 if self.is_enabled: raise VMProfError("vmprof is already enabled") - if not (1e-6 <= interval <= 1.0): + if not (1e-6 <= interval < 1.0): raise VMProfError("bad value for 'interval'") interval_usec = int(interval * 1000000.0) # self.fileno = fileno - self.is_enabled = True self._write_header(interval_usec) if not self.ever_enabled: if we_are_translated(): @@ -109,6 +108,22 @@ res = cintf.vmprof_enable(fileno, interval_usec) if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) + self.is_enabled = True + + def disable(self): + """Disable vmprof. + Raises VMProfError if something goes wrong. + """ + if not self.is_enabled: + raise VMProfError("vmprof is not enabled") + self.is_enabled = False + self._flush_codes() + self.fileno = -1 + if we_are_translated(): + # does not work untranslated + res = cintf.vmprof_disable() + if res < 0: + raise VMProfError(os.strerror(rposix.get_saved_errno())) def _write_code_registration(self, uid, name): b = self._current_codes diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -19,6 +19,15 @@ #include "rvmprof_getpc.h" #include "rvmprof_base.h" #include +#include +#include +#include +#include +#include +#include +#include +#include +#include /************************************************************/ @@ -56,7 +65,7 @@ /************************************************************/ -static long volatile ignore_signals = 0; +static long volatile ignore_signals = 1; RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored) @@ -70,3 +79,179 @@ _InterlockedExchange(&ignore_signals, (long)ignored); #endif } + + +/* ************************************************************* + * functions to write a profile file compatible with gperftools + * ************************************************************* + */ + +#define MARKER_STACKTRACE '\x01' +#define MARKER_VIRTUAL_IP '\x02' +#define MARKER_TRAILER '\x03' + +static int profile_file; +static long profile_interval_usec; +static char atfork_hook_installed = 0; + +static int _write_all(const void *buf, size_t bufsize) +{ + while (bufsize > 0) { + ssize_t count = write(profile_file, buf, bufsize); + if (count <= 0) + return -1; /* failed */ + buf += count; + bufsize -= count; + } + return 0; +} + +static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { + int saved_errno = errno; + /* + void* stack[MAX_STACK_DEPTH]; + stack[0] = GetPC((ucontext_t*)ucontext); + int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); + depth++; // To account for pc value in stack[0]; + prof_write_stacktrace(stack, depth, 1); + */ + errno = saved_errno; +} + + +/************************************************************/ + +static int install_sigprof_handler(void) +{ + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = sigprof_handler; + sa.sa_flags = SA_RESTART | SA_SIGINFO; + if (sigemptyset(&sa.sa_mask) == -1 || + sigaction(SIGPROF, &sa, NULL) == -1) + return -1; + return 0; +} + +static int remove_sigprof_handler(void) +{ + sighandler_t res = signal(SIGPROF, SIG_DFL); + if (res == SIG_ERR) + return -1; + return 0; +} + +static int install_sigprof_timer(void) +{ + static struct itimerval timer; + timer.it_interval.tv_sec = 0; + timer.it_interval.tv_usec = profile_interval_usec; + timer.it_value = timer.it_interval; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) + return -1; + return 0; +} + +static int remove_sigprof_timer(void) { + static struct itimerval timer; + timer.it_interval.tv_sec = 0; + timer.it_interval.tv_usec = 0; + timer.it_value.tv_sec = 0; + timer.it_value.tv_usec = 0; + if (setitimer(ITIMER_PROF, &timer, NULL) != 0) + return -1; + return 0; +} + +static void atfork_disable_timer(void) { + if (profile_interval_usec > 0) { + remove_sigprof_timer(); + } +} + +static void atfork_enable_timer(void) { + if (profile_interval_usec > 0) { + install_sigprof_timer(); + } +} + +static int install_pthread_atfork_hooks(void) { + /* this is needed to prevent the problems described there: + - http://code.google.com/p/gperftools/issues/detail?id=278 + - http://lists.debian.org/debian-glibc/2010/03/msg00161.html + + TL;DR: if the RSS of the process is large enough, the clone() syscall + will be interrupted by the SIGPROF before it can complete, then + retried, interrupted again and so on, in an endless loop. The + solution is to disable the timer around the fork, and re-enable it + only inside the parent. + */ + if (atfork_hook_installed) + return 0; + int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); + if (ret != 0) + return -1; + atfork_hook_installed = 1; + return 0; +} + +RPY_EXTERN +int rpython_vmprof_enable(int fd, long interval_usec) +{ + assert(fd >= 0); + assert(interval_usec > 0); + profile_file = fd; + profile_interval_usec = interval_usec; + + if (install_pthread_atfork_hooks() == -1) + return -1; + if (install_sigprof_handler() == -1) + return -1; + if (install_sigprof_timer() == -1) + return -1; + rpython_vmprof_ignore_signals(0); + return 0; +} + +RPY_EXTERN +int rpython_vmprof_disable(void) +{ + int srcfd; + char buf[4096]; + ssize_t size; + unsigned char marker = MARKER_TRAILER; + + rpython_vmprof_ignore_signals(1); + profile_interval_usec = 0; + + if (_write_all(&marker, 1) < 0) + return -1; + +#ifdef __linux__ + // copy /proc/PID/maps to the end of the profile file + sprintf(buf, "/proc/%d/maps", getpid()); + srcfd = open(buf, O_RDONLY); + if (srcfd < 0) + return -1; + + while ((size = read(srcfd, buf, sizeof buf)) > 0) { + _write_all(buf, size); + } + close(srcfd); +#else + // freebsd and mac +# error "REVIEW AND FIX ME" + sprintf(buf, "procstat -v %d", getpid()); + src = popen(buf, "r"); + if (!src) { + vmprof_error = "error calling procstat"; + return -1; + } + while ((size = fread(buf, 1, sizeof buf, src))) { + write(profile_file, buf, size); + } + pclose(src); +#endif + + return 0; +} diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,3 +1,5 @@ RPY_EXTERN char *rpython_vmprof_init(void); RPY_EXTERN void vmprof_ignore_signals(int); +RPY_EXTERN int rpython_vmprof_enable(int, long); +RPY_EXTERN int rpython_vmprof_disable(void); diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py --- a/rpython/rlib/rvmprof/test/test_rvmprof.py +++ b/rpython/rlib/rvmprof/test/test_rvmprof.py @@ -96,6 +96,8 @@ get_vmprof().enable(fd, 0.5) res = main(code, 5) assert res == 42 + get_vmprof().disable() + os.close(fd) return 0 assert f() == 0 From noreply at buildbot.pypy.org Sun Aug 2 18:21:51 2015 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 2 Aug 2015 18:21:51 +0200 (CEST) Subject: [pypy-commit] pypy default: test, fix for issue #2105 Message-ID: <20150802162151.338451C02DE@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r78746:f3e27c185636 Date: 2015-08-02 19:22 +0300 http://bitbucket.org/pypy/pypy/changeset/f3e27c185636/ Log: test, fix for issue #2105 diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -106,7 +106,7 @@ 'not find it' % (str(libpypy_c),)) binaries.append((libpypy_c, libpypy_name)) # - builddir = options.builddir + builddir = py.path.local(options.builddir) pypydir = builddir.ensure(name, dir=True) includedir = basedir.join('include') # Recursively copy all headers, shutil has only ignore diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -3,28 +3,31 @@ from pypy.conftest import pypydir from pypy.tool.release import package from pypy.module.sys.version import CPYTHON_VERSION +from rpython.tool.udir import udir import tarfile, zipfile, sys -def test_dir_structure(test='test'): - # make sure we have sort of pypy-c - if sys.platform == 'win32': - basename = 'pypy-c.exe' - rename_pypy_c = 'pypy-c' - exe_name_in_archive = 'pypy-c.exe' - else: - basename = 'pypy-c' - rename_pypy_c = 'pypy' - exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('goal', basename) - try: +class TestPackaging: + def setup_class(cls): + # make sure we have sort of pypy-c + if sys.platform == 'win32': + basename = 'pypy-c.exe' + cls.rename_pypy_c = 'pypy-c' + cls.exe_name_in_archive = 'pypy-c.exe' + else: + basename = 'pypy-c' + cls.rename_pypy_c = 'pypy' + cls.exe_name_in_archive = 'bin/pypy' + cls.pypy_c = py.path.local(pypydir).join('goal', basename) + + def test_dir_structure(self, test='test'): retval, builddir = package.package( '--without-cffi', str(py.path.local(pypydir).dirpath()), - test, rename_pypy_c, _fake=True) + test, self.rename_pypy_c, _fake=True) assert retval == 0 prefix = builddir.join(test) cpyver = '%d.%d' % CPYTHON_VERSION[:2] assert prefix.join('lib-python', cpyver, 'test').check() - assert prefix.join(exe_name_in_archive).check() + assert prefix.join(self.exe_name_in_archive).check() assert prefix.join('lib_pypy', 'syslog.py').check() assert not prefix.join('lib_pypy', 'py').check() assert not prefix.join('lib_pypy', 'ctypes_configure').check() @@ -36,7 +39,7 @@ else: th = tarfile.open(str(builddir.join('%s.tar.bz2' % test))) syslog = th.getmember('%s/lib_pypy/syslog.py' % test) - exe = th.getmember('%s/%s' % (test, exe_name_in_archive)) + exe = th.getmember('%s/%s' % (test, self.exe_name_in_archive)) assert syslog.mode == 0644 assert exe.mode == 0755 assert exe.uname == '' @@ -63,16 +66,22 @@ check_include('modsupport.h') check_include('pypy_decl.h') check_include('numpy/arrayobject.h') - finally: - pass # to keep the indentation -def test_with_zipfile_module(): - prev = package.USE_ZIPFILE_MODULE - try: - package.USE_ZIPFILE_MODULE = True - test_dir_structure(test='testzipfile') - finally: - package.USE_ZIPFILE_MODULE = prev + def test_options(self, test='testoptions'): + builddir = udir.ensure("build", dir=True) + retval, builddir = package.package( + '--without-cffi', '--builddir', str(builddir), + str(py.path.local(pypydir).dirpath()), + test, self.rename_pypy_c, _fake=True) + + def test_with_zipfile_module(self): + prev = package.USE_ZIPFILE_MODULE + try: + package.USE_ZIPFILE_MODULE = True + self.test_dir_structure(test='testzipfile') + finally: + package.USE_ZIPFILE_MODULE = prev + def test_fix_permissions(tmpdir): if sys.platform == 'win32': From noreply at buildbot.pypy.org Sun Aug 2 18:33:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 2 Aug 2015 18:33:44 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Forgot to call the remove_*() functions in disable() Message-ID: <20150802163344.144AE1C0557@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78747:2ea21e4d4d89 Date: 2015-08-02 17:49 +0200 http://bitbucket.org/pypy/pypy/changeset/2ea21e4d4d89/ Log: Forgot to call the remove_*() functions in disable() diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -90,8 +90,8 @@ #define MARKER_VIRTUAL_IP '\x02' #define MARKER_TRAILER '\x03' -static int profile_file; -static long profile_interval_usec; +static int profile_file = -1; +static long profile_interval_usec = 0; static char atfork_hook_installed = 0; static int _write_all(const void *buf, size_t bufsize) @@ -204,26 +204,27 @@ profile_interval_usec = interval_usec; if (install_pthread_atfork_hooks() == -1) - return -1; + goto error; if (install_sigprof_handler() == -1) - return -1; + goto error; if (install_sigprof_timer() == -1) - return -1; + goto error; rpython_vmprof_ignore_signals(0); return 0; + + error: + profile_file = -1; + profile_interval_usec = 0; + return -1; } -RPY_EXTERN -int rpython_vmprof_disable(void) +static int close_profile(void) { int srcfd; char buf[4096]; ssize_t size; unsigned char marker = MARKER_TRAILER; - rpython_vmprof_ignore_signals(1); - profile_interval_usec = 0; - if (_write_all(&marker, 1) < 0) return -1; @@ -235,7 +236,10 @@ return -1; while ((size = read(srcfd, buf, sizeof buf)) > 0) { - _write_all(buf, size); + if (_write_all(buf, size) < 0) { + close(srcfd); + return -1; + } } close(srcfd); #else @@ -253,5 +257,20 @@ pclose(src); #endif + /* don't close() the file descriptor from here */ + profile_file = -1; return 0; } + +RPY_EXTERN +int rpython_vmprof_disable(void) +{ + rpython_vmprof_ignore_signals(1); + profile_interval_usec = 0; + + if (remove_sigprof_timer() == -1) + return -1; + if (remove_sigprof_handler() == -1) + return -1; + return close_profile(); +} From noreply at buildbot.pypy.org Sun Aug 2 20:30:34 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Aug 2015 20:30:34 +0200 (CEST) Subject: [pypy-commit] pypy ufunc-reduce: Remove wrong docstring Message-ID: <20150802183034.439521C0557@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: ufunc-reduce Changeset: r78748:adf035e65bc8 Date: 2015-08-02 18:18 +0100 http://bitbucket.org/pypy/pypy/changeset/adf035e65bc8/ Log: Remove wrong docstring diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -168,58 +168,6 @@ @unwrap_spec(keepdims=bool) def descr_reduce(self, space, w_obj, w_axis=None, w_dtype=None, w_out=None, keepdims=False): - """reduce(...) - reduce(a, axis=0) - - Reduces `a`'s dimension by one, by applying ufunc along one axis. - - Let :math:`a.shape = (N_0, ..., N_i, ..., N_{M-1})`. Then - :math:`ufunc.reduce(a, axis=i)[k_0, ..,k_{i-1}, k_{i+1}, .., k_{M-1}]` = - the result of iterating `j` over :math:`range(N_i)`, cumulatively applying - ufunc to each :math:`a[k_0, ..,k_{i-1}, j, k_{i+1}, .., k_{M-1}]`. - For a one-dimensional array, reduce produces results equivalent to: - :: - - r = op.identity # op = ufunc - for i in xrange(len(A)): - r = op(r, A[i]) - return r - - For example, add.reduce() is equivalent to sum(). - - Parameters - ---------- - a : array_like - The array to act on. - axis : int, optional - The axis along which to apply the reduction. - - Examples - -------- - >>> np.multiply.reduce([2,3,5]) - 30 - - A multi-dimensional array example: - - >>> X = np.arange(8).reshape((2,2,2)) - >>> X - array([[[0, 1], - [2, 3]], - [[4, 5], - [6, 7]]]) - >>> np.add.reduce(X, 0) - array([[ 4, 6], - [ 8, 10]]) - >>> np.add.reduce(X) # confirm: default axis value is 0 - array([[ 4, 6], - [ 8, 10]]) - >>> np.add.reduce(X, 1) - array([[ 2, 4], - [10, 12]]) - >>> np.add.reduce(X, 2) - array([[ 1, 5], - [ 9, 13]]) - """ from pypy.module.micronumpy.ndarray import W_NDimArray if w_axis is None: w_axis = space.wrap(0) @@ -257,8 +205,6 @@ if x < 0 or x >= shapelen: raise oefmt(space.w_ValueError, "'axis' entry is out of bounds") axes[i] = x - - else: if space.isinstance_w(w_axis, space.w_tuple) and space.len_w(w_axis) == 1: w_axis = space.getitem(w_axis, space.wrap(0)) From noreply at buildbot.pypy.org Sun Aug 2 20:30:35 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Aug 2015 20:30:35 +0200 (CEST) Subject: [pypy-commit] pypy ufunc-reduce: Clean up shape handling in ufunc.accumulate() Message-ID: <20150802183035.7F2351C0FF7@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: ufunc-reduce Changeset: r78749:8586b501eab6 Date: 2015-08-02 19:30 +0100 http://bitbucket.org/pypy/pypy/changeset/8586b501eab6/ Log: Clean up shape handling in ufunc.accumulate() diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py --- a/pypy/module/micronumpy/test/test_ufuncs.py +++ b/pypy/module/micronumpy/test/test_ufuncs.py @@ -1338,6 +1338,26 @@ assert subtract.accumulate([True]*200).dtype == dtype('bool') assert divide.accumulate([True]*200).dtype == dtype('int8') + def test_accumulate_shapes(self): + import numpy as np + a = np.arange(6).reshape(2, 1, 3) + assert np.add.accumulate(a).shape == (2, 1, 3) + raises(ValueError, "np.add.accumulate(a, out=np.zeros((3, 1, 3)))") + raises(ValueError, "np.add.accumulate(a, out=np.zeros((2, 3)))") + raises(ValueError, "np.add.accumulate(a, out=np.zeros((2, 3, 1)))") + b = np.zeros((2, 1, 3)) + np.add.accumulate(a, out=b, axis=2) + assert b[0, 0, 2] == 3 + + def test_accumulate_shapes_2(self): + import sys + if '__pypy__' not in sys.builtin_module_names: + skip('PyPy-specific behavior in np.ufunc.accumulate') + import numpy as np + a = np.arange(6).reshape(2, 1, 3) + raises(ValueError, "np.add.accumulate(a, out=np.zeros((2, 1, 3, 2)))") + + def test_noncommutative_reduce_accumulate(self): import numpy as np tosubtract = np.arange(5) diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -247,33 +247,25 @@ axis = axes[0] assert axis >= 0 dtype = self.find_binop_type(space, dtype) - call__array_wrap__ = True + shape = obj_shape[:] + if out: + # There appears to be a lot of accidental complexity in what + # shapes cnumpy allows for out. + # We simply require out.shape == obj.shape + if out.get_shape() != obj_shape: + raise oefmt(space.w_ValueError, + "output parameter shape mismatch, expecting " + "[%s], got [%s]", + ",".join([str(x) for x in shape]), + ",".join([str(x) for x in out.get_shape()]), + ) + dtype = out.get_dtype() + call__array_wrap__ = False + else: + out = W_NDimArray.from_shape(space, shape, dtype, + w_instance=obj) + call__array_wrap__ = True if shapelen > 1: - shape = obj_shape[:] - if out: - # Test for shape agreement - # XXX maybe we need to do broadcasting here, although I must - # say I don't understand the details for axis reduce - if out.ndims() > len(shape): - raise oefmt(space.w_ValueError, - "output parameter for reduction operation %s " - "has too many dimensions", self.name) - elif out.ndims() < len(shape): - raise oefmt(space.w_ValueError, - "output parameter for reduction operation %s " - "does not have enough dimensions", self.name) - elif out.get_shape() != shape: - raise oefmt(space.w_ValueError, - "output parameter shape mismatch, expecting " - "[%s], got [%s]", - ",".join([str(x) for x in shape]), - ",".join([str(x) for x in out.get_shape()]), - ) - call__array_wrap__ = False - dtype = out.get_dtype() - else: - out = W_NDimArray.from_shape(space, shape, dtype, - w_instance=obj) if obj.get_size() == 0: if self.identity is not None: out.fill(space, self.identity.convert_to(space, dtype)) @@ -281,14 +273,6 @@ loop.do_accumulate(space, self.func, obj, dtype, axis, out, self.identity) else: - if out: - call__array_wrap__ = False - if out.get_shape() != [obj.get_size()]: - raise OperationError(space.w_ValueError, space.wrap( - "out of incompatible size")) - else: - out = W_NDimArray.from_shape(space, [obj.get_size()], dtype, - w_instance=obj) loop.compute_reduce_cumulative(space, obj, out, dtype, self.func, self.identity) if call__array_wrap__: From noreply at buildbot.pypy.org Sun Aug 2 21:17:05 2015 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 2 Aug 2015 21:17:05 +0200 (CEST) Subject: [pypy-commit] pypy ufunc-reduce: Clean up accumulate loops Message-ID: <20150802191705.0DF521C02DE@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: ufunc-reduce Changeset: r78750:1eb5d76e3f7b Date: 2015-08-02 20:17 +0100 http://bitbucket.org/pypy/pypy/changeset/1eb5d76e3f7b/ Log: Clean up accumulate loops diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -214,68 +214,68 @@ obj_state = obj_iter.next(obj_state) return cur_value -reduce_cum_driver = jit.JitDriver( - name='numpy_reduce_cum_driver', +accumulate_flat_driver = jit.JitDriver( + name='numpy_accumulate_flat', greens=['shapelen', 'func', 'dtype', 'out_dtype'], reds='auto') -def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity): - obj_iter, obj_state = obj.create_iter() +def accumulate_flat(space, func, w_arr, calc_dtype, out, identity): + arr_iter, arr_state = w_arr.create_iter() out_iter, out_state = out.create_iter() out_iter.track_index = False if identity is None: - cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) + cur_value = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) out_iter.setitem(out_state, cur_value) out_state = out_iter.next(out_state) - obj_state = obj_iter.next(obj_state) + arr_state = arr_iter.next(arr_state) else: cur_value = identity.convert_to(space, calc_dtype) - shapelen = len(obj.get_shape()) + shapelen = len(w_arr.get_shape()) out_dtype = out.get_dtype() - while not obj_iter.done(obj_state): - reduce_cum_driver.jit_merge_point( - shapelen=shapelen, func=func, - dtype=calc_dtype, out_dtype=out_dtype) - rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) - cur_value = func(calc_dtype, cur_value, rval) + while not arr_iter.done(arr_state): + accumulate_flat_driver.jit_merge_point( + shapelen=shapelen, func=func, dtype=calc_dtype, + out_dtype=out_dtype) + w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) + cur_value = func(calc_dtype, cur_value, w_item) out_iter.setitem(out_state, out_dtype.coerce(space, cur_value)) out_state = out_iter.next(out_state) - obj_state = obj_iter.next(obj_state) + arr_state = arr_iter.next(arr_state) -accumulate_driver = jit.JitDriver(name='numpy_accumulate_flat', - greens=['shapelen', 'func', 'dtype'], - reds='auto') +accumulate_driver = jit.JitDriver( + name='numpy_accumulate', + greens=['shapelen', 'func', 'calc_dtype'], reds='auto') -def do_accumulate(space, func, arr, dtype, axis, out, identity): +def accumulate(space, func, w_arr, axis, calc_dtype, out, identity): out_iter, out_state = out.create_iter() - obj_shape = arr.get_shape() - temp_shape = obj_shape[:axis] + obj_shape[axis + 1:] - temp = W_NDimArray.from_shape(space, temp_shape, dtype, w_instance=arr) - temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis) + arr_shape = w_arr.get_shape() + temp_shape = arr_shape[:axis] + arr_shape[axis + 1:] + temp = W_NDimArray.from_shape(space, temp_shape, calc_dtype, w_instance=w_arr) + temp_iter = AxisIter(temp.implementation, w_arr.get_shape(), axis) temp_state = temp_iter.reset() - arr_iter, arr_state = arr.create_iter() + arr_iter, arr_state = w_arr.create_iter() arr_iter.track_index = False if identity is not None: - identity = identity.convert_to(space, dtype) - shapelen = len(obj_shape) + identity = identity.convert_to(space, calc_dtype) + shapelen = len(arr_shape) while not out_iter.done(out_state): accumulate_driver.jit_merge_point(shapelen=shapelen, func=func, - dtype=dtype) - w_val = arr_iter.getitem(arr_state).convert_to(space, dtype) + calc_dtype=calc_dtype) + w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) arr_state = arr_iter.next(arr_state) out_indices = out_iter.indices(out_state) if out_indices[axis] == 0: if identity is not None: - w_val = func(dtype, identity, w_val) + w_item = func(calc_dtype, identity, w_item) else: - cur = temp_iter.getitem(temp_state) - w_val = func(dtype, cur, w_val) + cur_value = temp_iter.getitem(temp_state) + w_item = func(calc_dtype, cur_value, w_item) - out_iter.setitem(out_state, w_val) + out_iter.setitem(out_state, w_item) out_state = out_iter.next(out_state) - temp_iter.setitem(temp_state, w_val) + temp_iter.setitem(temp_state, w_item) temp_state = temp_iter.next(temp_state) return out diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -270,11 +270,11 @@ if self.identity is not None: out.fill(space, self.identity.convert_to(space, dtype)) return out - loop.do_accumulate(space, self.func, obj, dtype, axis, - out, self.identity) + loop.accumulate( + space, self.func, obj, axis, dtype, out, self.identity) else: - loop.compute_reduce_cumulative(space, obj, out, dtype, self.func, - self.identity) + loop.accumulate_flat( + space, self.func, obj, dtype, out, self.identity) if call__array_wrap__: out = space.call_method(obj, '__array_wrap__', out) return out From noreply at buildbot.pypy.org Mon Aug 3 08:24:56 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 3 Aug 2015 08:24:56 +0200 (CEST) Subject: [pypy-commit] pypy vecopt: could end in a finish operation, crashed the optimizer Message-ID: <20150803062456.381351C1185@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt Changeset: r78751:c83f61da5f2d Date: 2015-08-03 08:22 +0200 http://bitbucket.org/pypy/pypy/changeset/c83f61da5f2d/ Log: could end in a finish operation, crashed the optimizer diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -855,14 +855,13 @@ def append_loop(self, loop, all_target_tokens): # append e.g. the peeled loop to this loop! - label, jump = loop.operations[0], loop.operations[-1] - assert label.getopnum() == rop.LABEL - assert jump.getopnum() == rop.JUMP + jump = loop.operations[-1] + assert jump.getdescr() is not None target_token = None i = 0 # adds all target token until the one is found that jumps from the # last instruction to the label - while target_token is not jump.getdescr(): + while i < len(loop.operations) and target_token is not jump.getdescr(): # there is another label op = loop.operations[i] if op.getopnum() == rop.LABEL: From noreply at buildbot.pypy.org Mon Aug 3 15:55:11 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Mon, 3 Aug 2015 15:55:11 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c8-gcc: produce the perf-PID.map directly from pypy Message-ID: <20150803135511.CACB81C04BE@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8-gcc Changeset: r78752:e61b8f6ff5db Date: 2015-08-03 15:41 +0200 http://bitbucket.org/pypy/pypy/changeset/e61b8f6ff5db/ Log: produce the perf-PID.map directly from pypy Produce the jitted code to trace name mapping for perf directly in PyPy. Should probably become a profagent in the future. diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py --- a/rpython/jit/backend/llsupport/assembler.py +++ b/rpython/jit/backend/llsupport/assembler.py @@ -12,7 +12,7 @@ from rpython.rlib.objectmodel import specialize, compute_unique_id from rpython.rtyper.annlowlevel import cast_instance_to_gcref, llhelper from rpython.rtyper.lltypesystem import rffi, lltype - +from rpython.jit.backend.x86 import perf_map DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', # 'b'ridge, 'l'abel or # 'e'ntry point @@ -407,8 +407,11 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap return bool(gcrootmap) and not gcrootmap.is_shadow_stack +def debug_bridge(descr_number, rawstart, codeendpos): + perf_map.write_perf_map_entry( + "bridge out of Guard 0x%x" % r_uint(descr_number), + r_uint(rawstart), r_uint(rawstart + codeendpos)) -def debug_bridge(descr_number, rawstart, codeendpos): debug_start("jit-backend-addr") debug_print("bridge out of Guard 0x%x has address 0x%x to 0x%x" % (r_uint(descr_number), r_uint(rawstart), diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -38,6 +38,7 @@ from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import compute_unique_id from rpython.rlib import rstm, nonconst +from rpython.jit.backend.x86 import perf_map class Assembler386(BaseAssembler): @@ -629,9 +630,13 @@ self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE, rawstart) looptoken._ll_loop_code = looppos + rawstart + # + name = "Loop %d (%s)" % (looptoken.number, loopname) + perf_map.write_perf_map_entry( + name, r_uint(rawstart), r_uint(rawstart + full_size)) debug_start("jit-backend-addr") - debug_print("Loop %d (%s) has address 0x%x to 0x%x (bootstrap 0x%x)" % ( - looptoken.number, loopname, + debug_print("%s has address 0x%x to 0x%x (bootstrap 0x%x)" % ( + name, r_uint(rawstart + looppos), r_uint(rawstart + size_excluding_failure_stuff), r_uint(rawstart))) diff --git a/rpython/jit/backend/x86/perf_map.py b/rpython/jit/backend/x86/perf_map.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/perf_map.py @@ -0,0 +1,40 @@ +""" +Support for generating a perf map in /tmp/perf-PID.map +""" + +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +eci = ExternalCompilationInfo(separate_module_sources = [r""" +#include + +FILE *pypy_perf_map_file = NULL; + +RPY_EXPORTED void perf_map_write_entry( + const char *name, + size_t start_addr, + size_t end_addr) { + + if (!pypy_perf_map_file) { + char pmap_filename[100]; + snprintf(pmap_filename, 100, "/tmp/perf-%d.map", getpid()); + pypy_perf_map_file = fopen(pmap_filename, "w"); + } + + fprintf(pypy_perf_map_file, "%lx %lx %s\n", start_addr, + end_addr - start_addr, name); +} +"""]) + +_perf_map_write_entry = rffi.llexternal( + 'perf_map_write_entry', + [rffi.CCHARP , lltype.Unsigned, lltype.Unsigned], lltype.Void, + compilation_info=eci, + _nowrapper=True, transactionsafe=True) + +# ____________________________________________________________ + +def write_perf_map_entry(name, start_addr, end_addr): + # XXX: should be a profagent + with rffi.scoped_str2charp("JIT: " + name) as loopname: + _perf_map_write_entry(loopname, start_addr, end_addr) diff --git a/rpython/translator/stm/test/targetjit1.py b/rpython/translator/stm/test/targetjit1.py --- a/rpython/translator/stm/test/targetjit1.py +++ b/rpython/translator/stm/test/targetjit1.py @@ -125,9 +125,8 @@ def start_thread(args): bootstrapper.acquire(args) try: - rthread.gc_thread_prepare() # (this has no effect any more) ident = rthread.start_new_thread(bootstrapper.bootstrap, ()) - except Exception, e: + except Exception: bootstrapper.release() # normally called by the new thread raise return ident From noreply at buildbot.pypy.org Mon Aug 3 16:49:54 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 3 Aug 2015 16:49:54 +0200 (CEST) Subject: [pypy-commit] pypy vecopt: failargs can be none Message-ID: <20150803144954.50BD81C04BE@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt Changeset: r78753:6ef92e33b506 Date: 2015-08-03 16:49 +0200 http://bitbucket.org/pypy/pypy/changeset/6ef92e33b506/ Log: failargs can be none diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py --- a/rpython/jit/metainterp/optimizeopt/vectorize.py +++ b/rpython/jit/metainterp/optimizeopt/vectorize.py @@ -461,6 +461,8 @@ op = guard_node.getoperation() failargs = op.getfailargs() for i,arg in enumerate(failargs): + if arg is None: + continue accum = arg.getaccum() if accum: accum.save_to_descr(op.getdescr(),i) From noreply at buildbot.pypy.org Mon Aug 3 17:26:10 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Mon, 3 Aug 2015 17:26:10 +0200 (CEST) Subject: [pypy-commit] pypy vecopt: setting the orig label arguments in the test case Message-ID: <20150803152610.35F5B1C0FFE@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt Changeset: r78754:0359a9053df7 Date: 2015-08-03 17:26 +0200 http://bitbucket.org/pypy/pypy/changeset/0359a9053df7/ Log: setting the orig label arguments in the test case diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py --- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py @@ -57,6 +57,8 @@ metainterp_sd = FakeMetaInterpStaticData(self.cpu) jitdriver_sd = FakeJitDriverStaticData() opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0) + label_index = loop.find_first_index(rop.LABEL) + opt.orig_label_args = loop.operations[label_index].getarglist()[:] return opt def vectoroptimizer_unrolled(self, loop, unroll_factor = -1): @@ -960,6 +962,7 @@ """ opt=""" [p0,i0] + label(p0,i0) v3 = vec_int_expand(42) label(p0,i0,v3) i20 = int_add(i0, 1) @@ -989,6 +992,7 @@ """ opt=""" [p0,i0,f3] + label(p0,i0,f3) v3 = vec_float_expand(f3) label(p0,i0,f3,v3) i20 = int_add(i0, 1) @@ -1324,5 +1328,34 @@ opt = self.vectorize(self.parse_loop(trace)) self.debug_print_operations(opt.loop) + def test_bug1(self): + trace=""" + [p0, p1, p4, p6, p7, p9, p10, p11, p12, i18, p17, i19, p20, i21, p22, i23, i24, p25, i26, i27, p28, i29] + guard_early_exit(descr=) [p1, p0, p4, p6, p7, p9, p10, p11, p12, p17, i18] + guard_not_invalidated(descr=) [p1, p0, p9, p4, p6, p7, p10, p11, p12, p17, i18] + i30 = int_lt(i18, i19) + guard_true(i30, descr=) [p1, p0, p20, p4, p6, p7, p9, p10, p11, p12, p17, i18] + i31 = int_lt(i18, i21) + guard_true(i31, descr=) [p1, p0, p22, i18, p4, p6, p7, p9, p10, p11, p12, p17, None] + i33 = getarrayitem_raw(i23, i18, descr=chararraydescr) + i34 = int_lt(i18, i24) + guard_true(i34, descr=) [p1, p0, p25, i18, p4, p6, p7, p9, p10, p11, p12, p17, i33, None] + i35 = getarrayitem_raw(i26, i18, descr=chararraydescr) + i36 = int_add(i33, i35) + i37 = int_lt(i18, i27) + guard_true(i37, descr=) [p1, p0, p28, i18, p4, p6, p7, p9, p10, p11, p12, p17, i36, None, None] + i39 = int_signext(i36, 1) + i40 = int_ne(i39, i36) + guard_false(i40, descr=) [p1, p0, p28, i18, i36, i39, p4, p6, p7, p9, p10, p11, p12, p17, None, None, None] + setarrayitem_raw(i29, i18, i39, descr=chararraydescr) + i42 = int_add(i18, 1) + i44 = getfield_raw(140737351872096, descr=) + i46 = int_lt(i44, 0) + guard_false(i46, descr=) [p1, p0, p4, p6, p7, p9, p10, p11, p12, p17, i42, None, None, None] + jump(p0, p1, p4, p6, p7, p9, p10, p11, p12, i42, p17, i19, p20, i21, p22, i23, i24, p25, i26, i27, p28, i29) + """ + opt = self.vectorize(self.parse_loop(trace)) + self.debug_print_operations(opt.loop) + class TestLLtype(BaseTestVectorize, LLtypeMixin): pass From noreply at buildbot.pypy.org Mon Aug 3 19:26:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 3 Aug 2015 19:26:07 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Tweaks, and implementation of a wait-free, best-effort algorithm to Message-ID: <20150803172607.3D3121C0063@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78755:f99230e64cbc Date: 2015-08-03 19:26 +0200 http://bitbucket.org/pypy/pypy/changeset/f99230e64cbc/ Log: Tweaks, and implementation of a wait-free, best-effort algorithm to handle multiple threads (some possibly in signals) that all try to write chunks of data to the same file diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -35,6 +35,9 @@ vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT, compilation_info=eci, save_err=rffi.RFFI_SAVE_ERRNO) +vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf", + [rffi.CCHARP, rffi.LONG], + lltype.Void, compilation_info=eci) ## vmprof_register_virtual_function = rffi.llexternal( ## "vmprof_register_virtual_function", diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -8,6 +8,7 @@ from rpython.rtyper.lltypesystem import rffi MAX_CODES = 8000 +MAX_FUNC_NAME = 128 # ____________________________________________________________ @@ -29,7 +30,6 @@ def _cleanup_(self): self.is_enabled = False - self.ever_enabled = False self.fileno = -1 self._current_codes = None if sys.maxint == 2147483647: @@ -93,15 +93,13 @@ if not (1e-6 <= interval < 1.0): raise VMProfError("bad value for 'interval'") interval_usec = int(interval * 1000000.0) - # + + p_error = cintf.vmprof_init() + if p_error: + raise VMProfError(rffi.charp2str(p_error)) + self.fileno = fileno self._write_header(interval_usec) - if not self.ever_enabled: - if we_are_translated(): - p_error = cintf.vmprof_init() - if p_error: - raise VMProfError(rffi.charp2str(p_error)) - self.ever_enabled = True self._gather_all_code_objs() if we_are_translated(): # does not work untranslated @@ -126,6 +124,8 @@ raise VMProfError(os.strerror(rposix.get_saved_errno())) def _write_code_registration(self, uid, name): + if len(name) > MAX_FUNC_NAME: + name = name[:MAX_FUNC_NAME] b = self._current_codes if b is None: b = self._current_codes = StringBuilder() @@ -139,20 +139,7 @@ def _flush_codes(self): buf = self._current_codes.build() self._current_codes = None - self._carefully_write(buf) - - def _carefully_write(self, buf): - fd = self.fileno - assert fd >= 0 - if not buf: - return - cintf.vmprof_ignore_signals(True) - try: - while len(buf) > 0: - num = os.write(fd, buf) - buf = buf[num:] - finally: - cintf.vmprof_ignore_signals(False) + cintf.vmprof_write_buf(buf, len(buf)) def _write_header(self, interval_usec): b = StringBuilder() @@ -164,7 +151,8 @@ b.append('\x04') # interp name b.append(chr(len('pypy'))) b.append('pypy') - self._carefully_write(b.build()) + buf = b.build() + cintf.vmprof_write_buf(buf, len(buf)) def _write_long_to_string_builder(l, b): diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -1,3 +1,22 @@ +/* VMPROF + * + * statistical sampling profiler specifically designed to profile programs + * which run on a Virtual Machine and/or bytecode interpreter, such as Python, + * etc. + * + * The logic to dump the C stack traces is partly stolen from the code in + * gperftools. + * The file "getpc.h" has been entirely copied from gperftools. + * + * Tested only on gcc, linux, x86_64. + * + * Copyright (C) 2014-2015 + * Antonio Cuni - anto.cuni at gmail.com + * Maciej Fijalkowski - fijall at gmail.com + * Armin Rigo - arigo at tunes.org + * + */ + #define _GNU_SOURCE 1 @@ -16,8 +35,6 @@ #endif -#include "rvmprof_getpc.h" -#include "rvmprof_base.h" #include #include #include @@ -28,6 +45,9 @@ #include #include #include +#include "rvmprof_getpc.h" +#include "rvmprof_unwind.h" +#include "rvmprof_mt.h" /************************************************************/ @@ -57,6 +77,8 @@ if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) goto error; } + if (prepare_concurrent_bufs() < 0) + return "out of memory"; return NULL; error: @@ -86,6 +108,9 @@ * ************************************************************* */ +#define MAX_FUNC_NAME 128 +#define MAX_STACK_DEPTH ((SINGLE_BUF_SIZE / sizeof(void *)) - 4) + #define MARKER_STACKTRACE '\x01' #define MARKER_VIRTUAL_IP '\x02' #define MARKER_TRAILER '\x03' @@ -94,32 +119,26 @@ static long profile_interval_usec = 0; static char atfork_hook_installed = 0; -static int _write_all(const void *buf, size_t bufsize) -{ - while (bufsize > 0) { - ssize_t count = write(profile_file, buf, bufsize); - if (count <= 0) - return -1; /* failed */ - buf += count; - bufsize -= count; - } - return 0; -} static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { + if (ignore_signals) + return; int saved_errno = errno; - /* +#if 0 void* stack[MAX_STACK_DEPTH]; stack[0] = GetPC((ucontext_t*)ucontext); - int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); + int depth = get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext); depth++; // To account for pc value in stack[0]; prof_write_stacktrace(stack, depth, 1); - */ +#endif errno = saved_errno; } -/************************************************************/ +/* ************************************************************* + * the setup and teardown functions + * ************************************************************* + */ static int install_sigprof_handler(void) { @@ -218,6 +237,18 @@ return -1; } +static int _write_all(const void *buf, size_t bufsize) +{ + while (bufsize > 0) { + ssize_t count = write(profile_file, buf, bufsize); + if (count <= 0) + return -1; /* failed */ + buf += count; + bufsize -= count; + } + return 0; +} + static int close_profile(void) { int srcfd; @@ -272,5 +303,19 @@ return -1; if (remove_sigprof_handler() == -1) return -1; + shutdown_concurrent_bufs(profile_file); return close_profile(); } + +RPY_EXTERN +void rpython_vmprof_write_buf(char *buf, long size) +{ + struct profbuf_s *p = reserve_buffer(profile_file); + + if (size > SINGLE_BUF_SIZE) + size = SINGLE_BUF_SIZE; + memcpy(p->data, buf, size); + p->data_size = size; + + commit_buffer(profile_file, p); +} diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,5 +1,6 @@ RPY_EXTERN char *rpython_vmprof_init(void); -RPY_EXTERN void vmprof_ignore_signals(int); +RPY_EXTERN void rpython_vmprof_ignore_signals(int); RPY_EXTERN int rpython_vmprof_enable(int, long); RPY_EXTERN int rpython_vmprof_disable(void); +RPY_EXTERN void rpython_vmprof_write_buf(char *, long); diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/rvmprof_mt.h new file mode 100644 --- /dev/null +++ b/rpython/rlib/rvmprof/src/rvmprof_mt.h @@ -0,0 +1,156 @@ +/* Support for multithreaded write() operations */ + +#include +#include +#include + +#define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int)) +#define MAX_NUM_BUFFERS 32 + +#if defined(__i386__) || defined(__amd64__) + static inline void write_fence(void) { asm("" : : : "memory"); } +#else + static inline void write_fence(void) { __sync_synchronize(); } +#endif + + +#define PROFBUF_UNUSED 0 +#define PROFBUF_FILLING 1 +#define PROFBUF_READY 2 + + +struct profbuf_s { + unsigned int data_size; + unsigned int data_offset; + char data[SINGLE_BUF_SIZE]; +}; + +static char volatile profbuf_state[MAX_NUM_BUFFERS]; +static struct profbuf_s *profbuf_all_buffers = NULL; +static int volatile profbuf_write_lock = 2; + + +static int prepare_concurrent_bufs(void) +{ + assert(sizeof(struct profbuf_s) == 8192); + + if (profbuf_all_buffers != NULL) { + munmap(profbuf_all_buffers, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS); + profbuf_all_buffers = NULL; + } + profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (profbuf_all_buffers == MAP_FAILED) { + profbuf_all_buffers = NULL; + return -1; + } + memset((char *)profbuf_state, PROFBUF_UNUSED, sizeof(profbuf_state)); + profbuf_write_lock = 0; + return 0; +} + +static void _write_single_ready_buffer(int fd, long i) +{ + struct profbuf_s *p = &profbuf_all_buffers[i]; + ssize_t count = write(fd, p->data + p->data_offset, p->data_size); + if (count == p->data_size) { + profbuf_state[i] = PROFBUF_UNUSED; + } + else if (count > 0) { + p->data_offset += count; + p->data_size -= count; + } +} + +static void _write_ready_buffers(int fd) +{ + long i; + int has_write_lock = 0; + + for (i = 0; i < MAX_NUM_BUFFERS; i++) { + if (profbuf_state[i] == PROFBUF_READY) { + if (!has_write_lock) { + if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 1)) + return; /* can't acquire the write lock, give up */ + has_write_lock = 1; + } + _write_single_ready_buffer(fd, i); + } + } + if (has_write_lock) + profbuf_write_lock = 0; +} + +static struct profbuf_s *reserve_buffer(int fd) +{ + /* Tries to enter a region of code that fills one buffer. If + successful, returns the profbuf_s. It fails only if the + concurrent buffers are all busy (extreme multithreaded usage). + + This might call write() to emit the data sitting in + previously-prepared buffers. In case of write() error, the + error is ignored but unwritten data stays in the buffers. + */ + long i; + + _write_ready_buffers(fd); + + for (i = 0; i < MAX_NUM_BUFFERS; i++) { + if (profbuf_state[i] == PROFBUF_UNUSED && + __sync_bool_compare_and_swap(&profbuf_state[i], PROFBUF_UNUSED, + PROFBUF_FILLING)) { + struct profbuf_s *p = &profbuf_all_buffers[i]; + p->data_size = 0; + p->data_offset = 0; + return p; + } + } + /* no unused buffer found */ + return NULL; +} + +static void commit_buffer(int fd, struct profbuf_s *buf) +{ + /* Leaves a region of code that filled 'buf'. + + This might call write() to emit the data now ready. In case of + write() error, the error is ignored but unwritten data stays in + the buffers. + */ + + /* Make sure every thread sees the full content of 'buf' */ + write_fence(); + + /* Then set the 'ready' flag */ + long i = buf - profbuf_all_buffers; + assert(profbuf_state[i] == PROFBUF_FILLING); + profbuf_state[i] = PROFBUF_READY; + + if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 1)) { + /* can't acquire the write lock, ignore */ + } + else { + _write_single_ready_buffer(fd, i); + profbuf_write_lock = 0; + } +} + +static void shutdown_concurrent_bufs(int fd) +{ + retry: + usleep(1); + if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 2)) { + /* spin loop */ + goto retry; + } + + /* last attempt to flush buffers */ + int i; + for (i = 0; i < MAX_NUM_BUFFERS; i++) { + if (profbuf_state[i] == PROFBUF_READY) { + _write_single_ready_buffer(fd, i); + } + } +} diff --git a/rpython/rlib/rvmprof/src/rvmprof_base.h b/rpython/rlib/rvmprof/src/rvmprof_unwind.h rename from rpython/rlib/rvmprof/src/rvmprof_base.h rename to rpython/rlib/rvmprof/src/rvmprof_unwind.h From noreply at buildbot.pypy.org Tue Aug 4 11:33:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 4 Aug 2015 11:33:18 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: More care about write errors Message-ID: <20150804093318.7C66A1C14B4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78756:bf237a263451 Date: 2015-08-04 11:33 +0200 http://bitbucket.org/pypy/pypy/changeset/bf237a263451/ Log: More care about write errors diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -303,7 +303,8 @@ return -1; if (remove_sigprof_handler() == -1) return -1; - shutdown_concurrent_bufs(profile_file); + if (shutdown_concurrent_bufs(profile_file) < 0) + return -1; return close_profile(); } diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/rvmprof_mt.h --- a/rpython/rlib/rvmprof/src/rvmprof_mt.h +++ b/rpython/rlib/rvmprof/src/rvmprof_mt.h @@ -28,6 +28,7 @@ static char volatile profbuf_state[MAX_NUM_BUFFERS]; static struct profbuf_s *profbuf_all_buffers = NULL; static int volatile profbuf_write_lock = 2; +static long profbuf_pending_write; static int prepare_concurrent_bufs(void) @@ -48,20 +49,34 @@ } memset((char *)profbuf_state, PROFBUF_UNUSED, sizeof(profbuf_state)); profbuf_write_lock = 0; + profbuf_pending_write = -1; return 0; } -static void _write_single_ready_buffer(int fd, long i) +static int _write_single_ready_buffer(int fd, long i) { + int err; + if (profbuf_pending_write >= 0 && profbuf_pending_write != i) { + err = _write_single_ready_buffer(fd, profbuf_pending_write); + if (err < 0 || profbuf_pending_write >= 0) + return err; + } struct profbuf_s *p = &profbuf_all_buffers[i]; ssize_t count = write(fd, p->data + p->data_offset, p->data_size); if (count == p->data_size) { profbuf_state[i] = PROFBUF_UNUSED; + profbuf_pending_write = -1; } - else if (count > 0) { - p->data_offset += count; - p->data_size -= count; + else { + if (count > 0) { + p->data_offset += count; + p->data_size -= count; + } + profbuf_pending_write = i; + if (count < 0) + return -1; } + return 0; } static void _write_ready_buffers(int fd) @@ -76,7 +91,8 @@ return; /* can't acquire the write lock, give up */ has_write_lock = 1; } - _write_single_ready_buffer(fd, i); + if (_write_single_ready_buffer(fd, i) < 0) + break; } } if (has_write_lock) @@ -137,7 +153,7 @@ } } -static void shutdown_concurrent_bufs(int fd) +static int shutdown_concurrent_bufs(int fd) { retry: usleep(1); @@ -150,7 +166,9 @@ int i; for (i = 0; i < MAX_NUM_BUFFERS; i++) { if (profbuf_state[i] == PROFBUF_READY) { - _write_single_ready_buffer(fd, i); + if (_write_single_ready_buffer(fd, i) < 0) + return -1; } } + return 0; } From noreply at buildbot.pypy.org Tue Aug 4 11:39:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 4 Aug 2015 11:39:48 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: reserve_buffer() can return NULL Message-ID: <20150804093948.4D5841C146A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78757:1cf561fab385 Date: 2015-08-04 11:39 +0200 http://bitbucket.org/pypy/pypy/changeset/1cf561fab385/ Log: reserve_buffer() can return NULL diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -311,7 +311,13 @@ RPY_EXTERN void rpython_vmprof_write_buf(char *buf, long size) { - struct profbuf_s *p = reserve_buffer(profile_file); + struct profbuf_s *p; + + while ((p = reserve_buffer(profile_file)) == NULL) { + /* spin loop waiting for a buffer to be ready; should almost never + be the case */ + usleep(1); + } if (size > SINGLE_BUF_SIZE) size = SINGLE_BUF_SIZE; From noreply at buildbot.pypy.org Fri Aug 7 10:05:25 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 7 Aug 2015 10:05:25 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: hack differently Message-ID: <20150807080525.07BAE1C1C80@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78842:70d324743d57 Date: 2015-08-06 22:25 +0200 http://bitbucket.org/pypy/pypy/changeset/70d324743d57/ Log: hack differently diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -326,7 +326,7 @@ return op in self.inparg_dict def get_constant_box(self, box): - box = self.force_box(box) + box = self.get_box_replacement(box) if isinstance(box, Const): return box if (box.type == 'i' and box.get_forwarded() and diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -129,6 +129,10 @@ def optimize_CALL_PURE_I(self, op): # Step 1: check if all arguments are constant + for arg in op.getarglist(): + self.optimizer.force_box(arg) + # XXX hack to ensure that virtuals that are + # constant are presented that way result = self._can_optimize_call_pure(op) if result is not None: # this removes a CALL_PURE with all constant arguments. From noreply at buildbot.pypy.org Fri Aug 7 10:05:18 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 7 Aug 2015 10:05:18 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix one test in test_optimizebasic (with one still failing) Message-ID: <20150807080518.903EF1C0F00@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78839:ac9bb32198ac Date: 2015-08-06 22:09 +0200 http://bitbucket.org/pypy/pypy/changeset/ac9bb32198ac/ Log: fix one test in test_optimizebasic (with one still failing) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -265,6 +265,9 @@ self.optimizations = optimizations + def force_op_from_preamble(self, op): + return op + def replace_guard(self, op, value): assert isinstance(value, info.NonNullPtrInfo) if value.last_guard_pos == -1: @@ -323,7 +326,7 @@ return op in self.inparg_dict def get_constant_box(self, box): - box = self.get_box_replacement(box) + box = self.force_box(box) if isinstance(box, Const): return box if (box.type == 'i' and box.get_forwarded() and diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -147,13 +147,9 @@ return # replace CALL_PURE with just CALL - args = op.getarglist() opnum = OpHelpers.call_for_descr(op.getdescr()) newop = self.optimizer.replace_op_with(op, opnum) self.emit_operation(newop) - #if self.optimizer.emitting_dissabled: - # self.extra_call_pure.append(op) # XXX - #else: # don't move call_pure_with_exception in the short preamble... # issue #2015 From noreply at buildbot.pypy.org Fri Aug 7 10:05:20 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 7 Aug 2015 10:05:20 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: write a passing test Message-ID: <20150807080520.B0AEA1C150B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78840:baf938297d47 Date: 2015-08-06 22:11 +0200 http://bitbucket.org/pypy/pypy/changeset/baf938297d47/ Log: write a passing test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -5736,5 +5736,23 @@ """ self.optimize_loop(ops, expected) + def test_getarrayitem_gc_pure_not_invalidated(self): + ops = """ + [p0] + i1 = getarrayitem_gc_pure_i(p0, 1, descr=arraydescr) + escape_n(p0) + i2 = getarrayitem_gc_pure_i(p0, 1, descr=arraydescr) + escape_n(i2) + jump(p0) + """ + expected = """ + [p0] + i1 = getarrayitem_gc_pure_i(p0, 1, descr=arraydescr) + escape_n(p0) + escape_n(i1) + jump(p0) + """ + self.optimize_loop(ops, expected) + class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin): pass From noreply at buildbot.pypy.org Fri Aug 7 10:05:22 2015 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 7 Aug 2015 10:05:22 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: one more down Message-ID: <20150807080522.E680A1C1521@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78841:94553b2cdb44 Date: 2015-08-06 22:20 +0200 http://bitbucket.org/pypy/pypy/changeset/94553b2cdb44/ Log: one more down diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -172,9 +172,9 @@ if self.optimizer.optpure: self.optimizer.optpure.pure(opnum, result) - def pure_from_args(self, opnum, args, result): + def pure_from_args(self, opnum, args, result, descr=None): if self.optimizer.optpure: - self.optimizer.optpure.pure_from_args(opnum, args, result) + self.optimizer.optpure.pure_from_args(opnum, args, result, descr) def has_pure_result(self, opnum, args, descr): if self.optimizer.optpure: diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -199,9 +199,10 @@ recentops = self.getrecentops(opnum) recentops.add(op) - def pure_from_args(self, opnum, args, op): + def pure_from_args(self, opnum, args, op, descr=None): newop = ResOperation(opnum, - [self.get_box_replacement(arg) for arg in args]) + [self.get_box_replacement(arg) for arg in args], + descr=descr) newop.set_forwarded(op) self.pure(opnum, newop) diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -23,6 +23,9 @@ def getarg(self, i): return self.op.getarg(i) + def getdescr(self): + return self.op.getdescr() + def __repr__(self): return "Preamble(%r)" % (self.op,) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2320,8 +2320,8 @@ i5 = int_add(i3, i4) # setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, i4, descr=nextdescr) setarrayitem_gc(p3, 0, i5, descr=arraydescr) - setfield_gc(p1, i4, descr=nextdescr) escape_n() jump(p1, i1, i2, p3, i3) """ @@ -2332,8 +2332,8 @@ i5 = int_add(i3, i4) # setfield_gc(p1, i2, descr=valuedescr) + setfield_gc(p1, i4, descr=nextdescr) setarrayitem_gc(p3, 0, i5, descr=arraydescr) - setfield_gc(p1, i4, descr=nextdescr) escape_n() jump(p1, i1, i2, p3, i3) """ From noreply at buildbot.pypy.org Fri Aug 7 10:50:54 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Fri, 7 Aug 2015 10:50:54 +0200 (CEST) Subject: [pypy-commit] pypy vecopt: missing update pack of nodes scheduled some nodes as a non packed operation (wrong obviously) Message-ID: <20150807085054.20A5B1C0221@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt Changeset: r78843:e8a74e18fc6e Date: 2015-08-07 10:50 +0200 http://bitbucket.org/pypy/pypy/changeset/e8a74e18fc6e/ Log: missing update pack of nodes scheduled some nodes as a non packed operation (wrong obviously) diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py --- a/rpython/jit/metainterp/optimizeopt/dependency.py +++ b/rpython/jit/metainterp/optimizeopt/dependency.py @@ -939,9 +939,11 @@ return False, 0 # c = (self.constant - other.constant) - if isinstance(self.var, Const) and isinstance(other.var, Const): - return True, (self.var.value - other.var.value) - if self.var.same_box(other.var): + svar = self.var + ovar = other.var + if isinstance(svar, ConstInt) and isinstance(ovar, ConstInt): + return True, (svar.getint() - ovar.getint()) + if svar.same_box(ovar): return True, c return False, 0 diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py --- a/rpython/jit/metainterp/optimizeopt/schedule.py +++ b/rpython/jit/metainterp/optimizeopt/schedule.py @@ -83,9 +83,9 @@ for dep in node.provides()[:]: # COPY to = dep.to node.remove_edge_to(to) + nodes = self.schedulable_nodes if not to.emitted and to.depends_count() == 0: # sorts them by priority - nodes = self.schedulable_nodes i = len(nodes)-1 while i >= 0: itnode = nodes[i] @@ -97,7 +97,7 @@ # if they have the same priority, sort them # using the original position in the trace if itnode.getindex() < to.getindex(): - nodes.insert(i+1, to) + nodes.insert(i, to) break i -= 1 else: @@ -821,13 +821,11 @@ """ def __init__(self, ops, input_type, output_type): self.operations = ops - for i,node in enumerate(self.operations): - node.pack = self - node.pack_position = i self.accum = None self.input_type = input_type self.output_type = output_type assert self.input_type is not None or self.output_type is not None + self.update_pack_of_nodes() def opcount(self): return len(self.operations) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py --- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py @@ -1072,16 +1072,16 @@ [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18] guard_not_invalidated() [p38, p12, p9, p14, p39, i37, i44, f35, i40, p42, i43, f34, i28, p36, i41] i50 = int_add(i28, 1) - i46 = int_add(i44, 8) i48 = int_add(i41, 8) i51 = int_add(i37, 8) + i54 = int_add(i41, 16) + i46 = int_add(i44, 8) + i56 = int_add(i37, 16) i52 = int_ge(i50, i18) i637 = int_add(i28, 2) i638 = int_ge(i637, i18) guard_false(i638) [p38, p12, p9, p14, p39, i37, i44, f35, i40, p42, i43, f34, i28, p36, i41] i55 = int_add(i44, 16) - i54 = int_add(i41, 16) - i56 = int_add(i37, 16) i629 = int_add(i28, 2) i57 = int_ge(i637, i18) v61 = vec_raw_load(i21, i44, 2, descr=floatarraydescr) @@ -1173,10 +1173,10 @@ v233 = vec_cast_singlefloat_to_float(v232) v234 = vec_int_unpack(v232, 2, 2) v235 = vec_cast_singlefloat_to_float(v234) + v237 = vec_float_add(v231, v235) + v239 = vec_cast_float_to_singlefloat(v237) v236 = vec_float_add(v229, v233) - v237 = vec_float_add(v231, v235) v238 = vec_cast_float_to_singlefloat(v236) - v239 = vec_cast_float_to_singlefloat(v237) v240 = vec_float_pack(v238, v239, 2, 2) vec_raw_store(p2, i4, v240, descr=singlefloatarraydescr) jump(p0, p1, p2, i207, i500) diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py --- a/rpython/jit/metainterp/optimizeopt/vectorize.py +++ b/rpython/jit/metainterp/optimizeopt/vectorize.py @@ -409,13 +409,12 @@ # do not inc in rightmost_match_leftmost # this could miss some pack j += 1 - # set for each node to which pack it belongs - pack = self.packset.packs[i] - pack.update_pack_of_nodes() + i += 1 j = 0 - i += 1 if len_before == len(self.packset.packs): break + for pack in self.packset.packs: + pack.update_pack_of_nodes() if not we_are_translated(): From noreply at buildbot.pypy.org Fri Aug 7 11:00:01 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 7 Aug 2015 11:00:01 +0200 (CEST) Subject: [pypy-commit] stmgc c8-reshare-pages: more fixes everywhere Message-ID: <20150807090002.563B11C0F00@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-reshare-pages Changeset: r1917:bc807f9a977e Date: 2015-08-07 11:02 +0200 http://bitbucket.org/pypy/stmgc/changeset/bc807f9a977e/ Log: more fixes everywhere diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -63,9 +63,10 @@ stm_char *oslice = ((stm_char *)obj) + SLICE_OFFSET(undo->slice); uintptr_t current_page_num = ((uintptr_t)oslice) / 4096; + /* never import anything into READONLY pages */ + assert(get_page_status_in(my_segnum, current_page_num) != PAGE_READONLY); + if (pagenum == -1) { - /* XXX: what about PAGE_READONLY? check that the below line handles - this case correctly */ if (get_page_status_in(my_segnum, current_page_num) != PAGE_ACCESSIBLE) continue; } else if (pagenum != current_page_num) { @@ -170,15 +171,13 @@ assert(page_status == PAGE_NO_ACCESS || page_status == PAGE_READONLY); - /* make our page write-ready */ - page_mark_accessible(my_segnum, pagenum); + if (page_status == PAGE_READONLY) { + /* make our page write-ready */ + page_mark_accessible(my_segnum, pagenum); - if (page_status == PAGE_READONLY) { - /* copy from seg0 (XXX: can we depend on copy-on-write - of the kernel somehow?) */ - pagecopy(get_virtual_page(my_segnum, pagenum), - get_virtual_page(0, pagenum)); - + dprintf((" > found READONLY, make others NO_ACCESS\n")); + /* our READONLY copy *has* to have the current data, no + copy necessary */ /* make READONLY pages in other segments NO_ACCESS */ for (i = 1; i < NB_SEGMENTS; i++) { if (i == my_segnum) @@ -193,12 +192,19 @@ } /* find who has the most recent revision of our page */ + /* XXX: uh, *more* recent would be enough, right? */ int copy_from_segnum = -1; uint64_t most_recent_rev = 0; + bool was_readonly = false; for (i = 1; i < NB_SEGMENTS; i++) { if (i == my_segnum) continue; + if (!was_readonly && get_page_status_in(i, pagenum) == PAGE_READONLY) { + was_readonly = true; + break; + } + struct stm_commit_log_entry_s *log_entry; log_entry = get_priv_segment(i)->last_commit_log_entry; if (get_page_status_in(i, pagenum) != PAGE_NO_ACCESS @@ -209,10 +215,29 @@ } OPT_ASSERT(copy_from_segnum != my_segnum); + if (was_readonly) { + assert(page_status == PAGE_NO_ACCESS); + /* this case could be avoided by making all NO_ACCESS to READONLY + when resharing pages (XXX: better?). + We may go from NO_ACCESS->READONLY->ACCESSIBLE on write with + 2 SIGSEGV in a row.*/ + dprintf((" > make a previously NO_ACCESS page READONLY\n")); + page_mark_readonly(my_segnum, pagenum); + + release_all_privatization_locks(); + return; + } + + /* make our page write-ready */ + page_mark_accessible(my_segnum, pagenum); + /* account for this page now: XXX */ /* increment_total_allocated(4096); */ + if (copy_from_segnum == -1) { + dprintf((" > found newly allocated page: copy from seg0\n")); + /* this page is only accessible in the sharing segment seg0 so far (new allocation). We can thus simply mark it accessible here. */ pagecopy(get_virtual_page(my_segnum, pagenum), @@ -221,6 +246,8 @@ return; } + dprintf((" > import data from seg %d\n", copy_from_segnum)); + /* before copying anything, acquire modification locks from our and the other segment */ uint64_t to_lock = (1UL << copy_from_segnum); diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -227,10 +227,12 @@ if (get_page_status(pagenum)->by_segment == (PAGE_ACCESSIBLE << 0)) return; /* only accessible in seg0 */ + /* XXX: fast-path for all pages=READONLY */ + long i; for (i = 1; i < NB_SEGMENTS; i++){ if (get_page_status_in(i, pagenum) == PAGE_ACCESSIBLE) { - /* XXX: also do for PAGE_NO_ACCESS */ + /* XXX: also do for PAGE_NO_ACCESS? */ page_mark_readonly(i, pagenum); } } @@ -266,10 +268,8 @@ } } - /* XXXXXXXXXXXXXXX: necessary? */ - msync(stm_object_pages + END_NURSERY_PAGE*4096, - NB_SHARED_PAGES*4096, - MS_SYNC); /* MS_INVALIDATE| */ + /* msync() done when validating seg0 */ + /* Now loop over all pages and re-share them if possible. */ uintptr_t pagenum, endpagenum; pagenum = END_NURSERY_PAGE; /* starts after the nursery */ diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -769,6 +769,14 @@ } } + /* XXXXXXX: necessary?? AFAIK it is undefined if changes + to the MAP_SHARED in seg0 propagate to the READONLY + MAP_SHARED/MAP_PRIVATE in other segments + TODO: only used pages */ + msync(stm_object_pages + END_NURSERY_PAGE*4096, + NB_SHARED_PAGES*4096UL, + MS_INVALIDATE|MS_SYNC); + ensure_gs_register(original_num); } diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -71,26 +71,15 @@ dprintf(("page_mark_accessible(%lu) in seg:%ld\n", pagenum, segnum)); - if (page_status == PAGE_NO_ACCESS) { - if (mprotect(get_virtual_page(segnum, pagenum), 4096, PROT_READ | PROT_WRITE)) { - perror("mprotect"); - stm_fatalerror("mprotect failed! Consider running 'sysctl vm.max_map_count=16777216'"); - } - } else { - /* PAGE_READONLY requires renewing the mmap to MAP_PRIVATE */ - char *addr = get_virtual_page(segnum, pagenum); - char *res = mmap(addr, 4096UL, - PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_NORESERVE|MAP_ANONYMOUS|MAP_FIXED, - -1, 0); - if (res == MAP_FAILED) - stm_fatalerror("%s failed (mmap): %m", "page_mark_accessible"); + if (mprotect(get_virtual_page(segnum, pagenum), 4096, PROT_READ | PROT_WRITE)) { + perror("mprotect"); + stm_fatalerror("mprotect failed! Consider running 'sysctl vm.max_map_count=16777216'"); } /* set this flag *after* we un-protected it, because XXX later */ set_page_status_in(segnum, pagenum, PAGE_ACCESSIBLE); set_hint_modified_recently(pagenum); - dprintf(("RW(seg%ld, page%lu)\n", segnum, pagenum)); + dprintf(("RW(seg%ld, page %lu)\n", segnum, pagenum)); } static void page_mark_inaccessible(long segnum, uintptr_t pagenum) @@ -102,7 +91,7 @@ set_page_status_in(segnum, pagenum, PAGE_NO_ACCESS); - dprintf(("NONE(seg%ld, page%lu)\n", segnum, pagenum)); + dprintf(("NONE(seg%ld, page %lu)\n", segnum, pagenum)); char *addr = get_virtual_page(segnum, pagenum); madvise(addr, 4096, MADV_DONTNEED); if (mprotect(addr, 4096, PROT_NONE)) { @@ -115,13 +104,16 @@ static void page_mark_readonly(long segnum, uintptr_t pagenum) { /* mark readonly and share with seg0 */ - assert(_has_mutex()); - assert(segnum > 0 && get_page_status_in(segnum, pagenum) == PAGE_ACCESSIBLE); + assert(segnum > 0 && + (get_page_status_in(segnum, pagenum) == PAGE_ACCESSIBLE + || get_page_status_in(segnum, pagenum) == PAGE_NO_ACCESS)); dprintf(("page_mark_readonly(%lu) in seg:%ld\n", pagenum, segnum)); char *virt_addr = get_virtual_page(segnum, pagenum); madvise(virt_addr, 4096UL, MADV_DONTNEED); - /* XXX: does it matter if SHARED or PRIVATE? */ + /* XXX: does it matter if SHARED or PRIVATE? + IF MAP_SHARED, make sure page_mark_accessible doesn't simply mprotect() but also + mmap() as MAP_PRIVATE */ char *res = mmap(virt_addr, 4096UL, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_NORESERVE, @@ -132,5 +124,5 @@ set_page_status_in(segnum, pagenum, PAGE_READONLY); - dprintf(("RO(seg%ld, page%lu)\n", segnum, pagenum)); + dprintf(("RO(seg%ld, page %lu)\n", segnum, pagenum)); } diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c --- a/c8/stm/smallmalloc.c +++ b/c8/stm/smallmalloc.c @@ -305,7 +305,7 @@ inaccessible from all other segments again (except seg0) */ uintptr_t page = (baseptr - stm_object_pages) / 4096UL; for (i = 1; i < NB_SEGMENTS; i++) { - if (get_page_status_in(i, page) == PAGE_ACCESSIBLE) + if (get_page_status_in(i, page) != PAGE_NO_ACCESS) page_mark_inaccessible(i, page); } diff --git a/c8/test/support.py b/c8/test/support.py --- a/c8/test/support.py +++ b/c8/test/support.py @@ -760,7 +760,7 @@ return lib._stm_get_page_status(pagenum) def stm_is_accessible_page(pagenum): - return stm_get_page_status() == PAGE_ACCESSIBLE + return stm_get_page_status(pagenum) == PAGE_ACCESSIBLE def stm_get_hint_modified_recently(pagenum): return lib._stm_get_hint_modified_recently(pagenum) diff --git a/c8/test/test_resharing.py b/c8/test/test_resharing.py --- a/c8/test/test_resharing.py +++ b/c8/test/test_resharing.py @@ -79,8 +79,8 @@ self.commit_transaction() p1 = stm_get_obj_pages(lp1)[0] p2 = stm_get_obj_pages(lp2)[0] + self.switch(1) - self.start_transaction() # NO_ACCESS stays NO_ACCESS assert stm_get_page_status(p1) == PAGE_NO_ACCESS @@ -97,5 +97,93 @@ assert stm_get_page_status(p1) == PAGE_READONLY assert stm_get_page_status(p2) == PAGE_READONLY + self.switch(3) + self.start_transaction() + assert stm_get_page_status(p1) == PAGE_NO_ACCESS + assert stm_get_page_status(p2) == PAGE_NO_ACCESS stm_set_char(lp1, 'a') stm_set_char(lp2, 'b') + # NO_ACCESS becomes ACCESSIBLE in this segment + assert stm_get_page_status(p1) == PAGE_ACCESSIBLE + assert stm_get_page_status(p2) == PAGE_ACCESSIBLE + + self.switch(0) + # INACCESSIBLE in others + assert stm_get_page_status(p1) == PAGE_NO_ACCESS + assert stm_get_page_status(p2) == PAGE_NO_ACCESS + + self.switch(1) + # others + assert stm_get_page_status(p1) == PAGE_NO_ACCESS + assert stm_get_page_status(p2) == PAGE_NO_ACCESS + + + + + def test_resharing_more(self): + self.start_transaction() + lp1 = stm_allocate(16) + big = GC_LAST_SMALL_SIZE+64 + lp2 = stm_allocate(big) + self.push_roots([lp1, lp2]) + stm_minor_collect() + lp1, lp2 = self.pop_roots() + self.push_roots([lp1, lp2]) + self.commit_transaction() + p1 = stm_get_obj_pages(lp1)[0] + p2 = stm_get_obj_pages(lp2)[0] + + self.switch(1) + self.start_transaction() + # ACCESSIBLE becomes RO + stm_get_char(lp1) + stm_get_char(lp2) + assert stm_get_page_status(p1) == PAGE_ACCESSIBLE + assert stm_get_page_status(p2) == PAGE_ACCESSIBLE + stm_major_collect() + stm_major_collect() + stm_major_collect() + assert stm_get_page_status(p1) == PAGE_READONLY + assert stm_get_page_status(p2) == PAGE_READONLY + + self.switch(0) + # ACCESSIBLE becomes READONLY here too + self.start_transaction() + assert stm_get_page_status(p1) == PAGE_READONLY + assert stm_get_page_status(p2) == PAGE_READONLY + + # RO|RO|NO|NO + + self.switch(2) + self.start_transaction() + # just reading makes NO_ACCESS become READONLY + assert stm_get_page_status(p1) == PAGE_NO_ACCESS + assert stm_get_page_status(p2) == PAGE_NO_ACCESS + stm_get_char(lp1) + stm_get_char(lp2) + assert stm_get_page_status(p1) == PAGE_READONLY + assert stm_get_page_status(p2) == PAGE_READONLY + + # RO|RO|RO|NO + + self.switch(3) + self.start_transaction() + # still INACCESSIBLE in others + assert stm_get_page_status(p1) == PAGE_NO_ACCESS + assert stm_get_page_status(p2) == PAGE_NO_ACCESS + + self.switch(2) + # writing makes READONLY->ACCESSIBLE + stm_set_char(lp1, 'a') + stm_set_char(lp2, 'b') + assert stm_get_page_status(p1) == PAGE_ACCESSIBLE + assert stm_get_page_status(p2) == PAGE_ACCESSIBLE + + self.switch(1) + # others go from RO->NO_ACCESS + assert stm_get_page_status(p1) == PAGE_NO_ACCESS + assert stm_get_page_status(p2) == PAGE_NO_ACCESS + stm_set_char(lp1, 'a') + stm_set_char(lp2, 'b') + assert stm_get_page_status(p1) == PAGE_ACCESSIBLE + assert stm_get_page_status(p2) == PAGE_ACCESSIBLE From noreply at buildbot.pypy.org Fri Aug 7 11:14:12 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Fri, 7 Aug 2015 11:14:12 +0200 (CEST) Subject: [pypy-commit] stmgc c8-reshare-pages: fix allocate_preexisting() -> tests pass Message-ID: <20150807091412.BF8AC1C0622@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-reshare-pages Changeset: r1918:2b0614b6ae72 Date: 2015-08-07 11:17 +0200 http://bitbucket.org/pypy/stmgc/changeset/2b0614b6ae72/ Log: fix allocate_preexisting() -> tests pass diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -134,8 +134,16 @@ const char *src, uintptr_t size) { uintptr_t np = dest - get_segment_base(segnum); - if (get_page_status_in(segnum, np / 4096) != PAGE_NO_ACCESS) + uintptr_t page = np / 4096UL; + uint8_t status = get_page_status_in(segnum, page); + if (status == PAGE_ACCESSIBLE) { memcpy(dest, src, size); + } else if (status == PAGE_READONLY) { + /* we can't write to this page as we must not trigger a SIGSEGV */ + page_mark_inaccessible(segnum, page); + } else { + assert(status == PAGE_NO_ACCESS); + } } object_t *stm_allocate_preexisting(ssize_t size_rounded_up, @@ -149,7 +157,9 @@ memcpy(nobj_seg0, initial_data, size_rounded_up); ((struct object_s *)nobj_seg0)->stm_flags = GCFLAG_WRITE_BARRIER; - acquire_privatization_lock(STM_SEGMENT->segment_num); + /* XXX: not sure if we need to do better: acquire all privatization + locks in order to make READONLY pages NO_ACCESS in _fill_preexisting_slice */ + acquire_all_privatization_locks(); DEBUG_EXPECT_SEGFAULT(false); long j; @@ -176,7 +186,7 @@ } DEBUG_EXPECT_SEGFAULT(true); - release_privatization_lock(STM_SEGMENT->segment_num); + release_all_privatization_locks(); write_fence(); /* make sure 'nobj' is fully initialized from all threads here */ diff --git a/c8/stm/pages.c b/c8/stm/pages.c --- a/c8/stm/pages.c +++ b/c8/stm/pages.c @@ -63,11 +63,12 @@ static void page_mark_accessible(long segnum, uintptr_t pagenum) { +#ifndef NDEBUG uint8_t page_status = get_page_status_in(segnum, pagenum); - assert(segnum==0 || page_status == PAGE_NO_ACCESS || page_status == PAGE_READONLY); +#endif dprintf(("page_mark_accessible(%lu) in seg:%ld\n", pagenum, segnum)); From noreply at buildbot.pypy.org Fri Aug 7 11:43:52 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Fri, 7 Aug 2015 11:43:52 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: remove prints Message-ID: <20150807094352.7CFE31C0221@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78844:0f5e23676bc1 Date: 2015-08-07 11:43 +0200 http://bitbucket.org/pypy/pypy/changeset/0f5e23676bc1/ Log: remove prints diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -176,14 +176,8 @@ vprof = self.pycode.vprof if isinstance(value, W_IntObject): times = vprof.see_int(varindex, value.intval) - if times > 50 and not self.pycode.printed[varindex]: - self.pycode.printed[varindex] = True - print "COMMON VALUE:", self.pycode.co_name, varindex, self.pycode.co_varnames[varindex], value.intval else: times = self.pycode.vprof.see_object(varindex, value) - if times > 50 and not self.pycode.printed[varindex]: - self.pycode.printed[varindex] = True - print "COMMON VALUE:", self.pycode.co_name, varindex, self.pycode.co_varnames[varindex], self.space.str_w(self.space.repr(value)) def mark_as_escaped(self): From noreply at buildbot.pypy.org Fri Aug 7 11:43:54 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Fri, 7 Aug 2015 11:43:54 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: try to speed up the interpreter a bit Message-ID: <20150807094354.82E011C0221@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78845:61627a2881ff Date: 2015-08-07 11:43 +0200 http://bitbucket.org/pypy/pypy/changeset/61627a2881ff/ Log: try to speed up the interpreter a bit diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -174,6 +174,8 @@ if we_are_jitted(): return vprof = self.pycode.vprof + if vprof.frozen: + return if isinstance(value, W_IntObject): times = vprof.see_int(varindex, value.intval) else: diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -18,14 +18,17 @@ def see_int(self, index, value): if self.frozen: return 0 - if self.counters[index] < 0 and self.values_int[index] == value: - self.counters[index] -= 1 - return -self.counters[index] + count = self.counters[index] + if count < 0: + if self.values_int[index] == value: + new_count = count - 1 + self.counters[index] = new_count + return -new_count else: - self.values_int[index] = value - self.counters[index] = -1 self.values_wref[index] = dead_ref - return 1 + self.values_int[index] = value + self.counters[index] = -1 + return 1 def see_object(self, index, value): if self.frozen: @@ -34,15 +37,17 @@ self.values_wref[index] = dead_ref self.counters[index] = 0 return 0 - if self.values_wref[index]() is value: - assert self.counters[index] > 0 - self.counters[index] += 1 - return self.counters[index] + count = self.counters[index] + if count > 0: + if self.values_wref[index]() is value: + new_count = count + 1 + self.counters[index] = new_count + return new_count else: - self.values_wref[index] = ref(value) - self.counters[index] = 1 self.values_int[index] = -1 - return 1 + self.values_wref[index] = ref(value) + self.counters[index] = 1 + return 1 @jit.elidable def is_variable_constant(self, index): From noreply at buildbot.pypy.org Fri Aug 7 12:09:01 2015 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 7 Aug 2015 12:09:01 +0200 (CEST) Subject: [pypy-commit] pypy default: be more specific about the obscure api, fixes breakage Message-ID: <20150807100901.E57791C01F4@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r78846:5468f60b0932 Date: 2015-08-07 13:01 +0300 http://bitbucket.org/pypy/pypy/changeset/5468f60b0932/ Log: be more specific about the obscure api, fixes breakage diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -1031,7 +1031,8 @@ elif space.isinstance_w(w_dtype, space.w_tuple): w_dtype0 = space.getitem(w_dtype, space.wrap(0)) w_dtype1 = space.getitem(w_dtype, space.wrap(1)) - if space.isinstance_w(w_dtype0, space.w_type): + if space.isinstance_w(w_dtype0, space.w_type) and \ + space.isinstance_w(w_dtype1, space.w_list): #obscure api - (subclass, spec). Ignore the subclass return make_new_dtype(space, w_subtype, w_dtype1, alignment, copy=copy, w_shape=w_shape, w_metadata=w_metadata) From noreply at buildbot.pypy.org Fri Aug 7 12:25:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 7 Aug 2015 12:25:46 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Connect the two ways to temporarily disable interrupts Message-ID: <20150807102546.03A5B1C0362@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78847:4d382eb7b1b2 Date: 2015-08-07 10:57 +0200 http://bitbucket.org/pypy/pypy/changeset/4d382eb7b1b2/ Log: Connect the two ways to temporarily disable interrupts diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -5,20 +5,16 @@ # error "skiplist.c needs to be included before" #endif -volatile int pypy_codemap_currently_invalid = 0; - -void pypy_codemap_invalid_set(int value) -{ -#ifndef _MSC_VER - if (value) - __sync_lock_test_and_set(&pypy_codemap_currently_invalid, 1); - else - __sync_lock_release(&pypy_codemap_currently_invalid); +#ifdef RPYTHON_VMPROF +RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored); +static void pypy_codemap_invalid_set(int ignored) { + rpython_vmprof_ignore_signals(ignored); +} #else - _InterlockedExchange((long volatile *)&pypy_codemap_currently_invalid, - (long)value); +static void pypy_codemap_invalid_set(int ignored) { + /* nothing */ +} #endif -} /************************************************************/ diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -24,6 +24,7 @@ includes = ['rvmprof.h'], libraries = libs, separate_module_files = [SRC.join('rvmprof.c')], + post_include_bits=['#define RPYTHON_VMPROF\n'], ) eci = ExternalCompilationInfo(**eci_kwds) From noreply at buildbot.pypy.org Fri Aug 7 12:25:48 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 7 Aug 2015 12:25:48 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Fix the logic here. Previously, all functions being called before Message-ID: <20150807102548.2FB211C0362@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78848:7acf82e90804 Date: 2015-08-07 12:25 +0200 http://bitbucket.org/pypy/pypy/changeset/7acf82e90804/ Log: Fix the logic here. Previously, all functions being called before enable() would have a vmprof_unique_id of 0, and this 0 was stored in the stack. diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -13,10 +13,10 @@ 'error': 'space.fromcache(interp_vmprof.Cache).w_error', } - def setup_after_space_initialization(self): - # Force the __extend__ hacks and method replacements to occur - # early. Without this, for example, 'PyCode._init_ready' was - # already found by the annotator to be the original empty - # method, and the annotator doesn't notice that interp_vmprof.py - # (loaded later) replaces this method. - import pypy.module._vmprof.interp_vmprof + +# Force the __extend__ hacks and method replacements to occur +# early. Without this, for example, 'PyCode._init_ready' was +# already found by the annotator to be the original empty +# method, and the annotator doesn't notice that interp_vmprof.py +# (loaded later) replaces this method. +import pypy.module._vmprof.interp_vmprof diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -27,38 +27,38 @@ self._code_classes = set() self._gather_all_code_objs = lambda: None self._cleanup_() + if sys.maxint == 2147483647: + self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit + else: + self._code_unique_id = 0x7000000000000000 def _cleanup_(self): self.is_enabled = False self.fileno = -1 self._current_codes = None - if sys.maxint == 2147483647: - self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit - else: - self._code_unique_id = 0x7000000000000000 @specialize.argtype(1) def register_code(self, code, full_name_func): """Register the code object. Call when a new code object is made. """ - if self.is_enabled and code._vmprof_unique_id == 0: + if code._vmprof_unique_id == 0: uid = self._code_unique_id + 1 code._vmprof_unique_id = uid self._code_unique_id = uid - self._write_code_registration(uid, full_name_func(code)) + if self.is_enabled: + self._write_code_registration(uid, full_name_func(code)) @specialize.argtype(1) def get_unique_id(self, code): """Return the internal unique ID of a code object. Can only be called after register_code(). Call this in the jitdriver's - method 'get_unique_id(*greenkey)'. Always returns 0 if we - didn't call register_code_object_class() on the class. + method 'get_unique_id(*greenkey)'. The wrapper from __init__.py + always returns 0 if we didn't call register_code_object_class() + on the class. """ uid = code._vmprof_unique_id - if uid == 0: - uid = self._code_unique_id + 1 - code._vmprof_unique_id = uid - self._code_unique_id = uid + # 'uid == 0' can occur here if the code object was prebuilt, + # or if register_code() was not called for another reason. return uid def register_code_object_class(self, CodeClass, full_name_func): @@ -91,8 +91,9 @@ def gather_all_code_objs(): all_code_objs = rgc.do_get_objects(try_cast_to_code) for code in all_code_objs: - uid = self.get_unique_id(code) - self._write_code_registration(uid, full_name_func(code)) + uid = code._vmprof_unique_id + if uid != 0: + self._write_code_registration(uid, full_name_func(code)) prev() # make a chained list of the gather() functions for all # the types of code objects From noreply at buildbot.pypy.org Fri Aug 7 13:48:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 7 Aug 2015 13:48:04 +0200 (CEST) Subject: [pypy-commit] pypy vmprof-review: Translation fix? Message-ID: <20150807114804.1DEA51C0F00@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vmprof-review Changeset: r78849:aa7a0a00b0ca Date: 2015-08-07 12:46 +0200 http://bitbucket.org/pypy/pypy/changeset/aa7a0a00b0ca/ Log: Translation fix? diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py --- a/rpython/rlib/rvmprof/__init__.py +++ b/rpython/rlib/rvmprof/__init__.py @@ -19,9 +19,16 @@ @specialize.call_location() def get_unique_id(code): + """Return the internal unique ID of a code object. Can only be + called after register_code(). Call this in the jitdriver's + method 'get_unique_id(*greenkey)'. This always returns 0 if we + didn't call register_code_object_class() on the class. + """ assert code is not None if _was_registered(code.__class__): - return _get_vmprof().get_unique_id(code) + # '0' can occur here too, if the code object was prebuilt, + # or if register_code() was not called for another reason. + return code._vmprof_unique_id return 0 def enable(fileno, interval): diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -48,19 +48,6 @@ if self.is_enabled: self._write_code_registration(uid, full_name_func(code)) - @specialize.argtype(1) - def get_unique_id(self, code): - """Return the internal unique ID of a code object. Can only be - called after register_code(). Call this in the jitdriver's - method 'get_unique_id(*greenkey)'. The wrapper from __init__.py - always returns 0 if we didn't call register_code_object_class() - on the class. - """ - uid = code._vmprof_unique_id - # 'uid == 0' can occur here if the code object was prebuilt, - # or if register_code() was not called for another reason. - return uid - def register_code_object_class(self, CodeClass, full_name_func): """NOT_RPYTHON Register statically the class 'CodeClass' as containing user From noreply at buildbot.pypy.org Tue Aug 11 11:17:04 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 11:17:04 +0200 (CEST) Subject: [pypy-commit] pypy default: Fix for df216970b57e: we now expect ".intval" to be really an int, never a bool Message-ID: <20150811091704.0AD041C04C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78878:167c83453673 Date: 2015-08-11 10:13 +0100 http://bitbucket.org/pypy/pypy/changeset/167c83453673/ Log: Fix for df216970b57e: we now expect ".intval" to be really an int, never a bool diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py --- a/pypy/objspace/std/boolobject.py +++ b/pypy/objspace/std/boolobject.py @@ -13,7 +13,7 @@ class W_BoolObject(W_IntObject): def __init__(self, boolval): - self.intval = not not boolval + self.intval = int(not not boolval) def __nonzero__(self): raise Exception("you cannot do that, you must use space.is_true()") diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py --- a/pypy/objspace/std/test/test_boolobject.py +++ b/pypy/objspace/std/test/test_boolobject.py @@ -4,6 +4,10 @@ self.false = self.space.w_False self.wrap = self.space.wrap + def test_init(self): + assert (self.false.intval, type(self.false.intval)) == (0, int) + assert (self.true.intval, type(self.true.intval)) == (1, int) + def test_repr(self): assert self.space.eq_w(self.space.repr(self.true), self.wrap("True")) assert self.space.eq_w(self.space.repr(self.false), self.wrap("False")) From noreply at buildbot.pypy.org Tue Aug 11 11:23:23 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 11 Aug 2015 11:23:23 +0200 (CEST) Subject: [pypy-commit] pypy stmgc-c8-gcc: add a TODO item that I just found Message-ID: <20150811092323.B07911C0683@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: stmgc-c8-gcc Changeset: r78879:a91821ab8a4a Date: 2015-08-11 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/a91821ab8a4a/ Log: add a TODO item that I just found diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -1,3 +1,10 @@ +------------------------------------------------------------ + +the loop in + rstm.update_marker_num(intmask(next_instr) * 2 + 1) +can take up to 10% of dispatch_bytecode for non-JIT. +(in pyopcode.py) + ------------------------------------------------------------ fuse the two 32bit setfield_gc for stmflags & tid in the jit From noreply at buildbot.pypy.org Tue Aug 11 11:25:21 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 11 Aug 2015 11:25:21 +0200 (CEST) Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: start thinking about this Message-ID: <20150811092521.66E6A1C0683@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-faster-smallobj-sync Changeset: r1929:9b46edbce4f6 Date: 2015-08-11 11:28 +0200 http://bitbucket.org/pypy/stmgc/changeset/9b46edbce4f6/ Log: start thinking about this diff --git a/c8/TODO b/c8/TODO --- a/c8/TODO +++ b/c8/TODO @@ -1,7 +1,7 @@ - fix markers (e.g. become_inevitable doesn't seem to show up) -- improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c) +IN_PROGRESS: improve sync of small objs on commit (see FLAG_SYNC_LARGE in nursery.c) - reshare pages: make seg0 MAP_SHARED in order to re-share private pages during major GC diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h --- a/c8/stm/smallmalloc.h +++ b/c8/stm/smallmalloc.h @@ -49,6 +49,18 @@ */ struct small_malloc_data_s { struct small_free_loc_s *loc_free[GC_N_SMALL_REQUESTS]; + + /* lists the memory ranges of uncommitted/overflow objs that + need to be flushed to other segments on commit (like + large_overflow_objects) */ + struct list_s *uncommitted_ranges; + + /* List of smallobjs that were unreachable in a major GC. We + really free them only on commit/abort because we would + otherwise need to deal with holes generated in the + uncommitted_ranges during major GCs (and I can't think of + a non-quadratic algorithm to deal with this). */ + struct list_s *delayed_free_objs; }; From noreply at buildbot.pypy.org Tue Aug 11 12:00:58 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 12:00:58 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: merged default into vecopt (vecopt-merge) Message-ID: <20150811100058.BEFAE1C11D3@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78880:e7b774925efd Date: 2015-08-10 17:20 +0200 http://bitbucket.org/pypy/pypy/changeset/e7b774925efd/ Log: merged default into vecopt (vecopt-merge) diff too long, truncating to 2000 out of 30123 lines diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -528,12 +528,13 @@ # result, the parsing rules here are less strict. # -_LegalCharsPatt = r"[\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]" +_LegalKeyChars = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=" +_LegalValueChars = _LegalKeyChars + r"\[\]" _CookiePattern = re.compile( r"(?x)" # This is a Verbose pattern r"\s*" # Optional whitespace at start of cookie r"(?P" # Start of group 'key' - ""+ _LegalCharsPatt +"+?" # Any word of at least one letter, nongreedy + "["+ _LegalKeyChars +"]+?" # Any word of at least one letter, nongreedy r")" # End of group 'key' r"(" # Optional group: there may not be a value. r"\s*=\s*" # Equal Sign @@ -542,7 +543,7 @@ r"|" # or r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr r"|" # or - ""+ _LegalCharsPatt +"*" # Any word or empty string + "["+ _LegalValueChars +"]*" # Any word or empty string r")" # End of group 'val' r")?" # End of optional value group r"\s*" # Any number of spaces. diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py --- a/lib-python/2.7/SimpleHTTPServer.py +++ b/lib-python/2.7/SimpleHTTPServer.py @@ -14,6 +14,7 @@ import posixpath import BaseHTTPServer import urllib +import urlparse import cgi import sys import shutil @@ -68,10 +69,14 @@ path = self.translate_path(self.path) f = None if os.path.isdir(path): - if not self.path.endswith('/'): + parts = urlparse.urlsplit(self.path) + if not parts.path.endswith('/'): # redirect browser - doing basically what apache does self.send_response(301) - self.send_header("Location", self.path + "/") + new_parts = (parts[0], parts[1], parts[2] + '/', + parts[3], parts[4]) + new_url = urlparse.urlunsplit(new_parts) + self.send_header("Location", new_url) self.end_headers() return None for index in "index.html", "index.htm": diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py --- a/lib-python/2.7/_LWPCookieJar.py +++ b/lib-python/2.7/_LWPCookieJar.py @@ -18,7 +18,7 @@ iso2time, time2isoz) def lwp_cookie_str(cookie): - """Return string representation of Cookie in an the LWP cookie file format. + """Return string representation of Cookie in the LWP cookie file format. Actually, the format is extended a bit -- see module docstring. diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py --- a/lib-python/2.7/_abcoll.py +++ b/lib-python/2.7/_abcoll.py @@ -548,23 +548,25 @@ If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v In either case, this is followed by: for k, v in F.items(): D[k] = v ''' - if len(args) > 2: - raise TypeError("update() takes at most 2 positional " - "arguments ({} given)".format(len(args))) - elif not args: - raise TypeError("update() takes at least 1 argument (0 given)") + if not args: + raise TypeError("descriptor 'update' of 'MutableMapping' object " + "needs an argument") self = args[0] - other = args[1] if len(args) >= 2 else () - - if isinstance(other, Mapping): - for key in other: - self[key] = other[key] - elif hasattr(other, "keys"): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value + args = args[1:] + if len(args) > 1: + raise TypeError('update expected at most 1 arguments, got %d' % + len(args)) + if args: + other = args[0] + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value for key, value in kwds.items(): self[key] = value diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -25,8 +25,8 @@ DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes # NOTE: Base classes defined here are registered with the "official" ABCs -# defined in io.py. We don't use real inheritance though, because we don't -# want to inherit the C implementations. +# defined in io.py. We don't use real inheritance though, because we don't want +# to inherit the C implementations. class BlockingIOError(IOError): @@ -775,7 +775,7 @@ clsname = self.__class__.__name__ try: name = self.name - except AttributeError: + except Exception: return "<_pyio.{0}>".format(clsname) else: return "<_pyio.{0} name={1!r}>".format(clsname, name) @@ -1216,8 +1216,10 @@ return self.writer.flush() def close(self): - self.writer.close() - self.reader.close() + try: + self.writer.close() + finally: + self.reader.close() def isatty(self): return self.reader.isatty() or self.writer.isatty() @@ -1538,7 +1540,7 @@ def __repr__(self): try: name = self.name - except AttributeError: + except Exception: return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding) else: return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format( diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py --- a/lib-python/2.7/_strptime.py +++ b/lib-python/2.7/_strptime.py @@ -335,9 +335,9 @@ # though week_of_year = -1 week_of_year_start = -1 - # weekday and julian defaulted to -1 so as to signal need to calculate + # weekday and julian defaulted to None so as to signal need to calculate # values - weekday = julian = -1 + weekday = julian = None found_dict = found.groupdict() for group_key in found_dict.iterkeys(): # Directives not explicitly handled below: @@ -434,14 +434,14 @@ year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. - if julian == -1 and week_of_year != -1 and weekday != -1: + if julian is None and week_of_year != -1 and weekday is not None: week_starts_Mon = True if week_of_year_start == 0 else False julian = _calc_julian_from_U_or_W(year, week_of_year, weekday, week_starts_Mon) # Cannot pre-calculate datetime_date() since can change in Julian # calculation and thus could have different value for the day of the week # calculation. - if julian == -1: + if julian is None: # Need to add 1 to result since first day of the year is 1, not 0. julian = datetime_date(year, month, day).toordinal() - \ datetime_date(year, 1, 1).toordinal() + 1 @@ -451,7 +451,7 @@ year = datetime_result.year month = datetime_result.month day = datetime_result.day - if weekday == -1: + if weekday is None: weekday = datetime_date(year, month, day).weekday() if leap_year_fix: # the caller didn't supply a year but asked for Feb 29th. We couldn't diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py --- a/lib-python/2.7/aifc.py +++ b/lib-python/2.7/aifc.py @@ -357,10 +357,13 @@ self._soundpos = 0 def close(self): - if self._decomp: - self._decomp.CloseDecompressor() - self._decomp = None - self._file.close() + decomp = self._decomp + try: + if decomp: + self._decomp = None + decomp.CloseDecompressor() + finally: + self._file.close() def tell(self): return self._soundpos diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py --- a/lib-python/2.7/binhex.py +++ b/lib-python/2.7/binhex.py @@ -32,7 +32,8 @@ pass # States (what have we written) -[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3) +_DID_HEADER = 0 +_DID_DATA = 1 # Various constants REASONABLY_LARGE=32768 # Minimal amount we pass the rle-coder @@ -235,17 +236,22 @@ self._write(data) def close(self): - if self.state < _DID_DATA: - self.close_data() - if self.state != _DID_DATA: - raise Error, 'Close at the wrong time' - if self.rlen != 0: - raise Error, \ - "Incorrect resource-datasize, diff=%r" % (self.rlen,) - self._writecrc() - self.ofp.close() - self.state = None - del self.ofp + if self.state is None: + return + try: + if self.state < _DID_DATA: + self.close_data() + if self.state != _DID_DATA: + raise Error, 'Close at the wrong time' + if self.rlen != 0: + raise Error, \ + "Incorrect resource-datasize, diff=%r" % (self.rlen,) + self._writecrc() + finally: + self.state = None + ofp = self.ofp + del self.ofp + ofp.close() def binhex(inp, out): """(infilename, outfilename) - Create binhex-encoded copy of a file""" @@ -463,11 +469,15 @@ return self._read(n) def close(self): - if self.rlen: - dummy = self.read_rsrc(self.rlen) - self._checkcrc() - self.state = _DID_RSRC - self.ifp.close() + if self.state is None: + return + try: + if self.rlen: + dummy = self.read_rsrc(self.rlen) + self._checkcrc() + finally: + self.state = None + self.ifp.close() def hexbin(inp, out): """(infilename, outfilename) - Decode binhexed file""" diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py --- a/lib-python/2.7/bsddb/test/test_all.py +++ b/lib-python/2.7/bsddb/test/test_all.py @@ -412,9 +412,6 @@ def get_dbp(self) : return self._db - import string - string.letters=[chr(i) for i in xrange(65,91)] - bsddb._db.DBEnv_orig = bsddb._db.DBEnv bsddb._db.DB_orig = bsddb._db.DB if bsddb.db.version() <= (4, 3) : diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py --- a/lib-python/2.7/bsddb/test/test_basics.py +++ b/lib-python/2.7/bsddb/test/test_basics.py @@ -999,7 +999,7 @@ for x in "The quick brown fox jumped over the lazy dog".split(): d2.put(x, self.makeData(x)) - for x in string.letters: + for x in string.ascii_letters: d3.put(x, x*70) d1.sync() @@ -1047,7 +1047,7 @@ if verbose: print rec rec = c3.next() - self.assertEqual(count, len(string.letters)) + self.assertEqual(count, len(string.ascii_letters)) c1.close() diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py --- a/lib-python/2.7/bsddb/test/test_dbshelve.py +++ b/lib-python/2.7/bsddb/test/test_dbshelve.py @@ -59,7 +59,7 @@ return bytes(key, "iso8859-1") # 8 bits def populateDB(self, d): - for x in string.letters: + for x in string.ascii_letters: d[self.mk('S' + x)] = 10 * x # add a string d[self.mk('I' + x)] = ord(x) # add an integer d[self.mk('L' + x)] = [x] * 10 # add a list diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py --- a/lib-python/2.7/bsddb/test/test_get_none.py +++ b/lib-python/2.7/bsddb/test/test_get_none.py @@ -26,14 +26,14 @@ d.open(self.filename, db.DB_BTREE, db.DB_CREATE) d.set_get_returns_none(1) - for x in string.letters: + for x in string.ascii_letters: d.put(x, x * 40) data = d.get('bad key') self.assertEqual(data, None) - data = d.get(string.letters[0]) - self.assertEqual(data, string.letters[0]*40) + data = d.get(string.ascii_letters[0]) + self.assertEqual(data, string.ascii_letters[0]*40) count = 0 c = d.cursor() @@ -43,7 +43,7 @@ rec = c.next() self.assertEqual(rec, None) - self.assertEqual(count, len(string.letters)) + self.assertEqual(count, len(string.ascii_letters)) c.close() d.close() @@ -54,14 +54,14 @@ d.open(self.filename, db.DB_BTREE, db.DB_CREATE) d.set_get_returns_none(0) - for x in string.letters: + for x in string.ascii_letters: d.put(x, x * 40) self.assertRaises(db.DBNotFoundError, d.get, 'bad key') self.assertRaises(KeyError, d.get, 'bad key') - data = d.get(string.letters[0]) - self.assertEqual(data, string.letters[0]*40) + data = d.get(string.ascii_letters[0]) + self.assertEqual(data, string.ascii_letters[0]*40) count = 0 exceptionHappened = 0 @@ -77,7 +77,7 @@ self.assertNotEqual(rec, None) self.assertTrue(exceptionHappened) - self.assertEqual(count, len(string.letters)) + self.assertEqual(count, len(string.ascii_letters)) c.close() d.close() diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py --- a/lib-python/2.7/bsddb/test/test_queue.py +++ b/lib-python/2.7/bsddb/test/test_queue.py @@ -10,7 +10,6 @@ #---------------------------------------------------------------------- - at unittest.skip("fails on Windows; see issue 22943") class SimpleQueueTestCase(unittest.TestCase): def setUp(self): self.filename = get_new_database_path() @@ -37,17 +36,17 @@ print "before appends" + '-' * 30 pprint(d.stat()) - for x in string.letters: + for x in string.ascii_letters: d.append(x * 40) - self.assertEqual(len(d), len(string.letters)) + self.assertEqual(len(d), len(string.ascii_letters)) d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") - self.assertEqual(len(d), len(string.letters)+3) + self.assertEqual(len(d), len(string.ascii_letters)+3) if verbose: print "before close" + '-' * 30 @@ -108,17 +107,17 @@ print "before appends" + '-' * 30 pprint(d.stat()) - for x in string.letters: + for x in string.ascii_letters: d.append(x * 40) - self.assertEqual(len(d), len(string.letters)) + self.assertEqual(len(d), len(string.ascii_letters)) d.put(100, "some more data") d.put(101, "and some more ") d.put(75, "out of order") d.put(1, "replacement data") - self.assertEqual(len(d), len(string.letters)+3) + self.assertEqual(len(d), len(string.ascii_letters)+3) if verbose: print "before close" + '-' * 30 diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py --- a/lib-python/2.7/bsddb/test/test_recno.py +++ b/lib-python/2.7/bsddb/test/test_recno.py @@ -4,12 +4,11 @@ import os, sys import errno from pprint import pprint +import string import unittest from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path -letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - #---------------------------------------------------------------------- @@ -39,7 +38,7 @@ d.open(self.filename, db.DB_RECNO, db.DB_CREATE) - for x in letters: + for x in string.ascii_letters: recno = d.append(x * 60) self.assertIsInstance(recno, int) self.assertGreaterEqual(recno, 1) @@ -270,7 +269,7 @@ d.set_re_pad(45) # ...test both int and char d.open(self.filename, db.DB_RECNO, db.DB_CREATE) - for x in letters: + for x in string.ascii_letters: d.append(x * 35) # These will be padded d.append('.' * 40) # this one will be exact diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py --- a/lib-python/2.7/chunk.py +++ b/lib-python/2.7/chunk.py @@ -85,8 +85,10 @@ def close(self): if not self.closed: - self.skip() - self.closed = True + try: + self.skip() + finally: + self.closed = True def isatty(self): if self.closed: diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py --- a/lib-python/2.7/codecs.py +++ b/lib-python/2.7/codecs.py @@ -20,8 +20,14 @@ "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE", "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE", "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE", + "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder", + "StreamReader", "StreamWriter", + "StreamReaderWriter", "StreamRecoder", + "getencoder", "getdecoder", "getincrementalencoder", + "getincrementaldecoder", "getreader", "getwriter", + "encode", "decode", "iterencode", "iterdecode", "strict_errors", "ignore_errors", "replace_errors", - "xmlcharrefreplace_errors", + "xmlcharrefreplace_errors", "backslashreplace_errors", "register_error", "lookup_error"] ### Constants @@ -1051,7 +1057,7 @@ during translation. One example where this happens is cp875.py which decodes - multiple character to \u001a. + multiple character to \\u001a. """ m = {} diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -330,7 +330,7 @@ # http://code.activestate.com/recipes/259174/ # Knuth, TAOCP Vol. II section 4.6.3 - def __init__(self, iterable=None, **kwds): + def __init__(*args, **kwds): '''Create a new, empty Counter object. And if given, count elements from an input iterable. Or, initialize the count from another mapping of elements to their counts. @@ -341,8 +341,15 @@ >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' + if not args: + raise TypeError("descriptor '__init__' of 'Counter' object " + "needs an argument") + self = args[0] + args = args[1:] + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) super(Counter, self).__init__() - self.update(iterable, **kwds) + self.update(*args, **kwds) def __missing__(self, key): 'The count of elements not in the Counter is zero.' @@ -393,7 +400,7 @@ raise NotImplementedError( 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.') - def update(self, iterable=None, **kwds): + def update(*args, **kwds): '''Like dict.update() but add counts instead of replacing them. Source can be an iterable, a dictionary, or another Counter instance. @@ -413,6 +420,14 @@ # contexts. Instead, we implement straight-addition. Both the inputs # and outputs are allowed to contain zero and negative counts. + if not args: + raise TypeError("descriptor 'update' of 'Counter' object " + "needs an argument") + self = args[0] + args = args[1:] + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + iterable = args[0] if args else None if iterable is not None: if isinstance(iterable, Mapping): if self: @@ -428,7 +443,7 @@ if kwds: self.update(kwds) - def subtract(self, iterable=None, **kwds): + def subtract(*args, **kwds): '''Like dict.update() but subtracts counts instead of replacing them. Counts can be reduced below zero. Both the inputs and outputs are allowed to contain zero and negative counts. @@ -444,6 +459,14 @@ -1 ''' + if not args: + raise TypeError("descriptor 'subtract' of 'Counter' object " + "needs an argument") + self = args[0] + args = args[1:] + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + iterable = args[0] if args else None if iterable is not None: self_get = self.get if isinstance(iterable, Mapping): diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -464,26 +464,42 @@ for ns_header in ns_headers: pairs = [] version_set = False - for ii, param in enumerate(re.split(r";\s*", ns_header)): - param = param.rstrip() - if param == "": continue - if "=" not in param: - k, v = param, None - else: - k, v = re.split(r"\s*=\s*", param, 1) - k = k.lstrip() + + # XXX: The following does not strictly adhere to RFCs in that empty + # names and values are legal (the former will only appear once and will + # be overwritten if multiple occurrences are present). This is + # mostly to deal with backwards compatibility. + for ii, param in enumerate(ns_header.split(';')): + param = param.strip() + + key, sep, val = param.partition('=') + key = key.strip() + + if not key: + if ii == 0: + break + else: + continue + + # allow for a distinction between present and empty and missing + # altogether + val = val.strip() if sep else None + if ii != 0: - lc = k.lower() + lc = key.lower() if lc in known_attrs: - k = lc - if k == "version": + key = lc + + if key == "version": # This is an RFC 2109 cookie. - v = _strip_quotes(v) + if val is not None: + val = _strip_quotes(val) version_set = True - if k == "expires": + elif key == "expires": # convert expires date to seconds since epoch - v = http2time(_strip_quotes(v)) # None if invalid - pairs.append((k, v)) + if val is not None: + val = http2time(_strip_quotes(val)) # None if invalid + pairs.append((key, val)) if pairs: if not version_set: diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat --- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat +++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat @@ -1,1 +1,1 @@ -svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . +svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ . diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py --- a/lib-python/2.7/ctypes/test/test_find.py +++ b/lib-python/2.7/ctypes/test/test_find.py @@ -32,15 +32,24 @@ def setUp(self): self.gl = self.glu = self.gle = None if lib_gl: - self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) + try: + self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) + except OSError: + pass if lib_glu: - self.glu = CDLL(lib_glu, RTLD_GLOBAL) + try: + self.glu = CDLL(lib_glu, RTLD_GLOBAL) + except OSError: + pass if lib_gle: try: self.gle = CDLL(lib_gle) except OSError: pass + def tearDown(self): + self.gl = self.glu = self.gle = None + @unittest.skipUnless(lib_gl, 'lib_gl not available') def test_gl(self): if self.gl: diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py --- a/lib-python/2.7/ctypes/test/test_pickling.py +++ b/lib-python/2.7/ctypes/test/test_pickling.py @@ -15,9 +15,9 @@ class Y(X): _fields_ = [("str", c_char_p)] -class PickleTest(unittest.TestCase): +class PickleTest: def dumps(self, item): - return pickle.dumps(item) + return pickle.dumps(item, self.proto) def loads(self, item): return pickle.loads(item) @@ -72,17 +72,15 @@ @xfail def test_wchar(self): - pickle.dumps(c_char("x")) + self.dumps(c_char(b"x")) # Issue 5049 - pickle.dumps(c_wchar(u"x")) + self.dumps(c_wchar(u"x")) -class PickleTest_1(PickleTest): - def dumps(self, item): - return pickle.dumps(item, 1) - -class PickleTest_2(PickleTest): - def dumps(self, item): - return pickle.dumps(item, 2) +for proto in range(pickle.HIGHEST_PROTOCOL + 1): + name = 'PickleTest_%s' % proto + globals()[name] = type(name, + (PickleTest, unittest.TestCase), + {'proto': proto}) if __name__ == "__main__": unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py --- a/lib-python/2.7/ctypes/test/test_pointers.py +++ b/lib-python/2.7/ctypes/test/test_pointers.py @@ -7,8 +7,6 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float] python_types = [int, int, int, int, int, long, int, long, long, long, float, float] -LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) -large_string = 'T' * 2 ** 25 class PointersTestCase(unittest.TestCase): @@ -191,9 +189,11 @@ self.assertEqual(bool(mth), True) def test_pointer_type_name(self): + LargeNamedType = type('T' * 2 ** 25, (Structure,), {}) self.assertTrue(POINTER(LargeNamedType)) def test_pointer_type_str_name(self): + large_string = 'T' * 2 ** 25 self.assertTrue(POINTER(large_string)) if __name__ == '__main__': diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py --- a/lib-python/2.7/ctypes/util.py +++ b/lib-python/2.7/ctypes/util.py @@ -178,7 +178,7 @@ res = re.findall(expr, data) if not res: return _get_soname(_findLib_gcc(name)) - res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y))) + res.sort(key=_num_version) return res[-1] elif sys.platform == "sunos5": diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py --- a/lib-python/2.7/distutils/__init__.py +++ b/lib-python/2.7/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.9" +__version__ = "2.7.10" #--end constants-- diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py --- a/lib-python/2.7/distutils/command/check.py +++ b/lib-python/2.7/distutils/command/check.py @@ -126,7 +126,7 @@ """Returns warnings when the provided data doesn't compile.""" source_path = StringIO() parser = Parser() - settings = frontend.OptionParser().get_default_values() + settings = frontend.OptionParser(components=(Parser,)).get_default_values() settings.tab_width = 4 settings.pep_references = None settings.rfc_references = None @@ -142,8 +142,8 @@ document.note_source(source_path, -1) try: parser.parse(data, document) - except AttributeError: - reporter.messages.append((-1, 'Could not finish the parsing.', - '', {})) + except AttributeError as e: + reporter.messages.append( + (-1, 'Could not finish the parsing: %s.' % e, '', {})) return reporter.messages diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py --- a/lib-python/2.7/distutils/dir_util.py +++ b/lib-python/2.7/distutils/dir_util.py @@ -83,7 +83,7 @@ """Create all the empty directories under 'base_dir' needed to put 'files' there. - 'base_dir' is just the a name of a directory which doesn't necessarily + 'base_dir' is just the name of a directory which doesn't necessarily exist yet; 'files' is a list of filenames to be interpreted relative to 'base_dir'. 'base_dir' + the directory portion of every file in 'files' will be created if it doesn't already exist. 'mode', 'verbose' and diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py --- a/lib-python/2.7/distutils/tests/test_check.py +++ b/lib-python/2.7/distutils/tests/test_check.py @@ -1,5 +1,6 @@ # -*- encoding: utf8 -*- """Tests for distutils.command.check.""" +import textwrap import unittest from test.test_support import run_unittest @@ -93,6 +94,36 @@ cmd = self._run(metadata, strict=1, restructuredtext=1) self.assertEqual(cmd._warnings, 0) + @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils") + def test_check_restructuredtext_with_syntax_highlight(self): + # Don't fail if there is a `code` or `code-block` directive + + example_rst_docs = [] + example_rst_docs.append(textwrap.dedent("""\ + Here's some code: + + .. code:: python + + def foo(): + pass + """)) + example_rst_docs.append(textwrap.dedent("""\ + Here's some code: + + .. code-block:: python + + def foo(): + pass + """)) + + for rest_with_code in example_rst_docs: + pkg_info, dist = self.create_dist(long_description=rest_with_code) + cmd = check(dist) + cmd.check_restructuredtext() + self.assertEqual(cmd._warnings, 0) + msgs = cmd._check_rst_data(rest_with_code) + self.assertEqual(len(msgs), 0) + def test_check_all(self): metadata = {'url': 'xxx', 'author': 'xxx'} diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py --- a/lib-python/2.7/distutils/text_file.py +++ b/lib-python/2.7/distutils/text_file.py @@ -124,11 +124,11 @@ def close (self): """Close the current file and forget everything we know about it (filename, current line number).""" - - self.file.close () + file = self.file self.file = None self.filename = None self.current_line = None + file.close() def gen_error (self, msg, line=None): diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py --- a/lib-python/2.7/dumbdbm.py +++ b/lib-python/2.7/dumbdbm.py @@ -21,6 +21,7 @@ """ +import ast as _ast import os as _os import __builtin__ import UserDict @@ -85,7 +86,7 @@ with f: for line in f: line = line.rstrip() - key, pos_and_siz_pair = eval(line) + key, pos_and_siz_pair = _ast.literal_eval(line) self._index[key] = pos_and_siz_pair # Write the index dict to the directory file. The original directory @@ -208,8 +209,10 @@ return len(self._index) def close(self): - self._commit() - self._index = self._datfile = self._dirfile = self._bakfile = None + try: + self._commit() + finally: + self._index = self._datfile = self._dirfile = self._bakfile = None __del__ = close diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py --- a/lib-python/2.7/encodings/uu_codec.py +++ b/lib-python/2.7/encodings/uu_codec.py @@ -84,7 +84,7 @@ data = a2b_uu(s) except binascii.Error, v: # Workaround for broken uuencoders by /Fredrik Lundh - nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3 + nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3 data = a2b_uu(s[:nbytes]) #sys.stderr.write("Warning: %s\n" % str(v)) write(data) diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py --- a/lib-python/2.7/ensurepip/__init__.py +++ b/lib-python/2.7/ensurepip/__init__.py @@ -12,9 +12,9 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "7.0" +_SETUPTOOLS_VERSION = "15.2" -_PIP_VERSION = "1.5.6" +_PIP_VERSION = "6.1.1" # pip currently requires ssl support, so we try to provide a nicer # error message when that is missing (http://bugs.python.org/issue19744) diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl deleted file mode 100644 Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171 GIT binary patch [cut] diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl deleted file mode 100644 Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py --- a/lib-python/2.7/fileinput.py +++ b/lib-python/2.7/fileinput.py @@ -233,8 +233,10 @@ self.close() def close(self): - self.nextfile() - self._files = () + try: + self.nextfile() + finally: + self._files = () def __iter__(self): return self @@ -270,23 +272,25 @@ output = self._output self._output = 0 - if output: - output.close() + try: + if output: + output.close() + finally: + file = self._file + self._file = 0 + try: + if file and not self._isstdin: + file.close() + finally: + backupfilename = self._backupfilename + self._backupfilename = 0 + if backupfilename and not self._backup: + try: os.unlink(backupfilename) + except OSError: pass - file = self._file - self._file = 0 - if file and not self._isstdin: - file.close() - - backupfilename = self._backupfilename - self._backupfilename = 0 - if backupfilename and not self._backup: - try: os.unlink(backupfilename) - except OSError: pass - - self._isstdin = False - self._buffer = [] - self._bufindex = 0 + self._isstdin = False + self._buffer = [] + self._bufindex = 0 def readline(self): try: diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py --- a/lib-python/2.7/fnmatch.py +++ b/lib-python/2.7/fnmatch.py @@ -47,12 +47,14 @@ import os,posixpath result=[] pat=os.path.normcase(pat) - if not pat in _cache: + try: + re_pat = _cache[pat] + except KeyError: res = translate(pat) if len(_cache) >= _MAXCACHE: _cache.clear() - _cache[pat] = re.compile(res) - match=_cache[pat].match + _cache[pat] = re_pat = re.compile(res) + match = re_pat.match if os.path is posixpath: # normcase on posix is NOP. Optimize it away from the loop. for name in names: @@ -71,12 +73,14 @@ its arguments. """ - if not pat in _cache: + try: + re_pat = _cache[pat] + except KeyError: res = translate(pat) if len(_cache) >= _MAXCACHE: _cache.clear() - _cache[pat] = re.compile(res) - return _cache[pat].match(name) is not None + _cache[pat] = re_pat = re.compile(res) + return re_pat.match(name) is not None def translate(pat): """Translate a shell PATTERN to a regular expression. diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py --- a/lib-python/2.7/ftplib.py +++ b/lib-python/2.7/ftplib.py @@ -594,11 +594,16 @@ def close(self): '''Close the connection without assuming anything about it.''' - if self.file is not None: - self.file.close() - if self.sock is not None: - self.sock.close() - self.file = self.sock = None + try: + file = self.file + self.file = None + if file is not None: + file.close() + finally: + sock = self.sock + self.sock = None + if sock is not None: + sock.close() try: import ssl @@ -638,12 +643,24 @@ '221 Goodbye.' >>> ''' - ssl_version = ssl.PROTOCOL_TLSv1 + ssl_version = ssl.PROTOCOL_SSLv23 def __init__(self, host='', user='', passwd='', acct='', keyfile=None, - certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): + certfile=None, context=None, + timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None): + if context is not None and keyfile is not None: + raise ValueError("context and keyfile arguments are mutually " + "exclusive") + if context is not None and certfile is not None: + raise ValueError("context and certfile arguments are mutually " + "exclusive") self.keyfile = keyfile self.certfile = certfile + if context is None: + context = ssl._create_stdlib_context(self.ssl_version, + certfile=certfile, + keyfile=keyfile) + self.context = context self._prot_p = False FTP.__init__(self, host, user, passwd, acct, timeout) @@ -656,12 +673,12 @@ '''Set up secure control connection by using TLS/SSL.''' if isinstance(self.sock, ssl.SSLSocket): raise ValueError("Already using TLS") - if self.ssl_version == ssl.PROTOCOL_TLSv1: + if self.ssl_version >= ssl.PROTOCOL_SSLv23: resp = self.voidcmd('AUTH TLS') else: resp = self.voidcmd('AUTH SSL') - self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile, - ssl_version=self.ssl_version) + self.sock = self.context.wrap_socket(self.sock, + server_hostname=self.host) self.file = self.sock.makefile(mode='rb') return resp @@ -692,8 +709,8 @@ def ntransfercmd(self, cmd, rest=None): conn, size = FTP.ntransfercmd(self, cmd, rest) if self._prot_p: - conn = ssl.wrap_socket(conn, self.keyfile, self.certfile, - ssl_version=self.ssl_version) + conn = self.context.wrap_socket(conn, + server_hostname=self.host) return conn, size def retrbinary(self, cmd, callback, blocksize=8192, rest=None): diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py --- a/lib-python/2.7/genericpath.py +++ b/lib-python/2.7/genericpath.py @@ -10,6 +10,14 @@ 'getsize', 'isdir', 'isfile'] +try: + _unicode = unicode +except NameError: + # If Python is built without Unicode support, the unicode type + # will not exist. Fake one. + class _unicode(object): + pass + # Does a path exist? # This is false for dangling symbolic links on systems that support them. def exists(path): diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py --- a/lib-python/2.7/gettext.py +++ b/lib-python/2.7/gettext.py @@ -52,7 +52,9 @@ __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog', 'find', 'translation', 'install', 'textdomain', 'bindtextdomain', - 'dgettext', 'dngettext', 'gettext', 'ngettext', + 'bind_textdomain_codeset', + 'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext', + 'ldngettext', 'lngettext', 'ngettext', ] _default_localedir = os.path.join(sys.prefix, 'share', 'locale') @@ -294,11 +296,12 @@ # See if we're looking at GNU .mo conventions for metadata if mlen == 0: # Catalog description - lastk = k = None + lastk = None for item in tmsg.splitlines(): item = item.strip() if not item: continue + k = v = None if ':' in item: k, v = item.split(':', 1) k = k.strip().lower() diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py --- a/lib-python/2.7/gzip.py +++ b/lib-python/2.7/gzip.py @@ -238,9 +238,9 @@ data = data.tobytes() if len(data) > 0: - self.size = self.size + len(data) + self.fileobj.write(self.compress.compress(data)) + self.size += len(data) self.crc = zlib.crc32(data, self.crc) & 0xffffffffL - self.fileobj.write( self.compress.compress(data) ) self.offset += len(data) return len(data) @@ -369,19 +369,21 @@ return self.fileobj is None def close(self): - if self.fileobj is None: + fileobj = self.fileobj + if fileobj is None: return - if self.mode == WRITE: - self.fileobj.write(self.compress.flush()) - write32u(self.fileobj, self.crc) - # self.size may exceed 2GB, or even 4GB - write32u(self.fileobj, self.size & 0xffffffffL) - self.fileobj = None - elif self.mode == READ: - self.fileobj = None - if self.myfileobj: - self.myfileobj.close() - self.myfileobj = None + self.fileobj = None + try: + if self.mode == WRITE: + fileobj.write(self.compress.flush()) + write32u(fileobj, self.crc) + # self.size may exceed 2GB, or even 4GB + write32u(fileobj, self.size & 0xffffffffL) + finally: + myfileobj = self.myfileobj + if myfileobj: + self.myfileobj = None + myfileobj.close() def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH): self._check_closed() diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py --- a/lib-python/2.7/hashlib.py +++ b/lib-python/2.7/hashlib.py @@ -187,7 +187,7 @@ def prf(msg, inner=inner, outer=outer): # PBKDF2_HMAC uses the password as key. We can re-use the same - # digest objects and and just update copies to skip initialization. + # digest objects and just update copies to skip initialization. icpy = inner.copy() ocpy = outer.copy() icpy.update(msg) diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py --- a/lib-python/2.7/htmlentitydefs.py +++ b/lib-python/2.7/htmlentitydefs.py @@ -1,6 +1,6 @@ """HTML character entity references.""" -# maps the HTML entity name to the Unicode codepoint +# maps the HTML entity name to the Unicode code point name2codepoint = { 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 @@ -256,7 +256,7 @@ 'zwnj': 0x200c, # zero width non-joiner, U+200C NEW RFC 2070 } -# maps the Unicode codepoint to the HTML entity name +# maps the Unicode code point to the HTML entity name codepoint2name = {} # maps the HTML entity name to the character diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py --- a/lib-python/2.7/httplib.py +++ b/lib-python/2.7/httplib.py @@ -68,6 +68,7 @@ from array import array import os +import re import socket from sys import py3kwarning from urlparse import urlsplit @@ -218,6 +219,38 @@ # maximum amount of headers accepted _MAXHEADERS = 100 +# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) +# +# VCHAR = %x21-7E +# obs-text = %x80-FF +# header-field = field-name ":" OWS field-value OWS +# field-name = token +# field-value = *( field-content / obs-fold ) +# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] +# field-vchar = VCHAR / obs-text +# +# obs-fold = CRLF 1*( SP / HTAB ) +# ; obsolete line folding +# ; see Section 3.2.4 + +# token = 1*tchar +# +# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" +# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" +# / DIGIT / ALPHA +# ; any VCHAR, except delimiters +# +# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1 + +# the patterns for both name and value are more leniant than RFC +# definitions to allow for backwards compatibility +_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match +_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search + +# We always set the Content-Length header for these methods because some +# servers will otherwise respond with a 411 +_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} + class HTTPMessage(mimetools.Message): @@ -313,6 +346,11 @@ hlist.append(line) self.addheader(headerseen, line[len(headerseen)+1:].strip()) continue + elif headerseen is not None: + # An empty header name. These aren't allowed in HTTP, but it's + # probably a benign mistake. Don't add the header, just keep + # going. + continue else: # It's not a header line; throw it back and stop here. if not self.dict: @@ -522,9 +560,10 @@ return True def close(self): - if self.fp: - self.fp.close() + fp = self.fp + if fp: self.fp = None + fp.close() def isclosed(self): # NOTE: it is possible that we will not ever call self.close(). This @@ -723,7 +762,7 @@ endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT request to the proxy server when the connection is established. - This method must be called before the HTML connection has been + This method must be called before the HTTP connection has been established. The headers argument should be a mapping of extra HTTP headers @@ -797,13 +836,17 @@ def close(self): """Close the connection to the HTTP server.""" - if self.sock: - self.sock.close() # close it manually... there may be other refs - self.sock = None - if self.__response: - self.__response.close() - self.__response = None self.__state = _CS_IDLE + try: + sock = self.sock + if sock: + self.sock = None + sock.close() # close it manually... there may be other refs + finally: + response = self.__response + if response: + self.__response = None + response.close() def send(self, data): """Send `data' to the server.""" @@ -978,7 +1021,16 @@ if self.__state != _CS_REQ_STARTED: raise CannotSendHeader() - hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values])) + header = '%s' % header + if not _is_legal_header_name(header): + raise ValueError('Invalid header name %r' % (header,)) + + values = [str(v) for v in values] + for one_value in values: + if _is_illegal_header_value(one_value): + raise ValueError('Invalid header value %r' % (one_value,)) + + hdr = '%s: %s' % (header, '\r\n\t'.join(values)) self._output(hdr) def endheaders(self, message_body=None): @@ -1000,19 +1052,25 @@ """Send a complete request to the server.""" self._send_request(method, url, body, headers) - def _set_content_length(self, body): - # Set the content-length based on the body. + def _set_content_length(self, body, method): + # Set the content-length based on the body. If the body is "empty", we + # set Content-Length: 0 for methods that expect a body (RFC 7230, + # Section 3.3.2). If the body is set for other methods, we set the + # header provided we can figure out what the length is. thelen = None - try: - thelen = str(len(body)) - except TypeError, te: - # If this is a file-like object, try to - # fstat its file descriptor + if body is None and method.upper() in _METHODS_EXPECTING_BODY: + thelen = '0' + elif body is not None: try: - thelen = str(os.fstat(body.fileno()).st_size) - except (AttributeError, OSError): - # Don't send a length if this failed - if self.debuglevel > 0: print "Cannot stat!!" + thelen = str(len(body)) + except TypeError: + # If this is a file-like object, try to + # fstat its file descriptor + try: + thelen = str(os.fstat(body.fileno()).st_size) + except (AttributeError, OSError): + # Don't send a length if this failed + if self.debuglevel > 0: print "Cannot stat!!" if thelen is not None: self.putheader('Content-Length', thelen) @@ -1028,8 +1086,8 @@ self.putrequest(method, url, **skips) - if body is not None and 'content-length' not in header_names: - self._set_content_length(body) + if 'content-length' not in header_names: + self._set_content_length(body, method) for hdr, value in headers.iteritems(): self.putheader(hdr, value) self.endheaders(body) @@ -1072,20 +1130,20 @@ try: response.begin() + assert response.will_close != _UNKNOWN + self.__state = _CS_IDLE + + if response.will_close: + # this effectively passes the connection to the response + self.close() + else: + # remember this, so we can tell when it is complete + self.__response = response + + return response except: response.close() raise - assert response.will_close != _UNKNOWN - self.__state = _CS_IDLE - - if response.will_close: - # this effectively passes the connection to the response - self.close() - else: - # remember this, so we can tell when it is complete - self.__response = response - - return response class HTTP: @@ -1129,7 +1187,7 @@ "Accept arguments to set the host/port, since the superclass doesn't." if host is not None: - self._conn._set_hostport(host, port) + (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port) self._conn.connect() def getfile(self): diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py --- a/lib-python/2.7/idlelib/CodeContext.py +++ b/lib-python/2.7/idlelib/CodeContext.py @@ -15,8 +15,8 @@ from sys import maxint as INFINITY from idlelib.configHandler import idleConf -BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with"]) +BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", + "if", "try", "while", "with"} UPDATEINTERVAL = 100 # millisec FONTUPDATEINTERVAL = 1000 # millisec diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py --- a/lib-python/2.7/idlelib/EditorWindow.py +++ b/lib-python/2.7/idlelib/EditorWindow.py @@ -469,13 +469,10 @@ ("format", "F_ormat"), ("run", "_Run"), ("options", "_Options"), - ("windows", "_Windows"), + ("windows", "_Window"), ("help", "_Help"), ] - if sys.platform == "darwin": - menu_specs[-2] = ("windows", "_Window") - def createmenubar(self): mbar = self.menubar diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py --- a/lib-python/2.7/idlelib/FormatParagraph.py +++ b/lib-python/2.7/idlelib/FormatParagraph.py @@ -44,9 +44,11 @@ The length limit parameter is for testing with a known value. """ - if limit == None: + if limit is None: + # The default length limit is that defined by pep8 limit = idleConf.GetOption( - 'main', 'FormatParagraph', 'paragraph', type='int') + 'extensions', 'FormatParagraph', 'max-width', + type='int', default=72) text = self.editwin.text first, last = self.editwin.get_selection_indices() if first and last: diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py --- a/lib-python/2.7/idlelib/PyShell.py +++ b/lib-python/2.7/idlelib/PyShell.py @@ -871,13 +871,10 @@ ("edit", "_Edit"), ("debug", "_Debug"), ("options", "_Options"), - ("windows", "_Windows"), + ("windows", "_Window"), ("help", "_Help"), ] - if sys.platform == "darwin": - menu_specs[-2] = ("windows", "_Window") - # New classes from idlelib.IdleHistory import History @@ -1350,7 +1347,7 @@ if type(s) not in (unicode, str, bytearray): # See issue #19481 if isinstance(s, unicode): - s = unicode.__getslice__(s, None, None) + s = unicode.__getitem__(s, slice(None)) elif isinstance(s, str): s = str.__str__(s) elif isinstance(s, bytearray): diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py --- a/lib-python/2.7/idlelib/SearchEngine.py +++ b/lib-python/2.7/idlelib/SearchEngine.py @@ -191,7 +191,7 @@ This is done by searching forwards until there is no match. Prog: compiled re object with a search method returning a match. - Chars: line of text, without \n. + Chars: line of text, without \\n. Col: stop index for the search; the limit for match.end(). ''' m = prog.search(chars) diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def --- a/lib-python/2.7/idlelib/config-extensions.def +++ b/lib-python/2.7/idlelib/config-extensions.def @@ -66,6 +66,7 @@ [FormatParagraph] enable=True +max-width=72 [FormatParagraph_cfgBindings] format-paragraph= diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def --- a/lib-python/2.7/idlelib/config-main.def +++ b/lib-python/2.7/idlelib/config-main.def @@ -58,9 +58,6 @@ font-bold= 0 encoding= none -[FormatParagraph] -paragraph=72 - [Indent] use-spaces= 1 num-spaces= 4 diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py --- a/lib-python/2.7/idlelib/configDialog.py +++ b/lib-python/2.7/idlelib/configDialog.py @@ -371,7 +371,6 @@ parent = self.parent self.winWidth = StringVar(parent) self.winHeight = StringVar(parent) - self.paraWidth = StringVar(parent) self.startupEdit = IntVar(parent) self.autoSave = IntVar(parent) self.encoding = StringVar(parent) @@ -387,7 +386,6 @@ frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Autosave Preferences ') frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE) - frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE) frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE) frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE, text=' Additional Help Sources ') @@ -416,11 +414,6 @@ labelWinHeightTitle = Label(frameWinSize, text='Height') entryWinHeight = Entry( frameWinSize, textvariable=self.winHeight, width=3) - #paragraphFormatWidth - labelParaWidthTitle = Label( - frameParaSize, text='Paragraph reformat width (in characters)') - entryParaWidth = Entry( - frameParaSize, textvariable=self.paraWidth, width=3) #frameEncoding labelEncodingTitle = Label( frameEncoding, text="Default Source Encoding") @@ -458,7 +451,6 @@ frameRun.pack(side=TOP, padx=5, pady=5, fill=X) frameSave.pack(side=TOP, padx=5, pady=5, fill=X) frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X) - frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X) frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X) frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) #frameRun @@ -475,9 +467,6 @@ labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5) entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5) labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5) - #paragraphFormatWidth - labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) - entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5) #frameEncoding labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5) radioEncNone.pack(side=RIGHT, anchor=E, pady=5) @@ -509,7 +498,6 @@ self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin) self.winWidth.trace_variable('w', self.VarChanged_winWidth) self.winHeight.trace_variable('w', self.VarChanged_winHeight) - self.paraWidth.trace_variable('w', self.VarChanged_paraWidth) self.startupEdit.trace_variable('w', self.VarChanged_startupEdit) self.autoSave.trace_variable('w', self.VarChanged_autoSave) self.encoding.trace_variable('w', self.VarChanged_encoding) @@ -594,10 +582,6 @@ value = self.winHeight.get() self.AddChangedItem('main', 'EditorWindow', 'height', value) - def VarChanged_paraWidth(self, *params): - value = self.paraWidth.get() - self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value) - def VarChanged_startupEdit(self, *params): value = self.startupEdit.get() self.AddChangedItem('main', 'General', 'editor-on-startup', value) @@ -1094,9 +1078,6 @@ 'main', 'EditorWindow', 'width', type='int')) self.winHeight.set(idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int')) - #initial paragraph reformat size - self.paraWidth.set(idleConf.GetOption( - 'main', 'FormatParagraph', 'paragraph', type='int')) # default source encoding self.encoding.set(idleConf.GetOption( 'main', 'EditorWindow', 'encoding', default='none')) diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt --- a/lib-python/2.7/idlelib/help.txt +++ b/lib-python/2.7/idlelib/help.txt @@ -100,7 +100,7 @@ which is scrolling off the top or the window. (Not present in Shell window.) -Windows Menu: +Window Menu: Zoom Height -- toggles the window between configured size and maximum height. diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat --- a/lib-python/2.7/idlelib/idle.bat +++ b/lib-python/2.7/idlelib/idle.bat @@ -1,4 +1,4 @@ - at echo off -rem Start IDLE using the appropriate Python interpreter -set CURRDIR=%~dp0 -start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 + at echo off +rem Start IDLE using the appropriate Python interpreter +set CURRDIR=%~dp0 +start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9 diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py --- a/lib-python/2.7/idlelib/idle_test/test_calltips.py +++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py @@ -55,7 +55,8 @@ def gtest(obj, out): self.assertEqual(signature(obj), out) - gtest(List, '()\n' + List.__doc__) + if List.__doc__ is not None: + gtest(List, '()\n' + List.__doc__) gtest(list.__new__, 'T.__new__(S, ...) -> a new object with type S, a subtype of T') gtest(list.__init__, @@ -70,7 +71,8 @@ def test_signature_wrap(self): # This is also a test of an old-style class - self.assertEqual(signature(textwrap.TextWrapper), '''\ + if textwrap.TextWrapper.__doc__ is not None: + self.assertEqual(signature(textwrap.TextWrapper), '''\ (width=70, initial_indent='', subsequent_indent='', expand_tabs=True, replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, drop_whitespace=True, break_on_hyphens=True)''') @@ -106,20 +108,23 @@ def t5(a, b=None, *args, **kwds): 'doc' t5.tip = "(a, b=None, *args, **kwargs)" + doc = '\ndoc' if t1.__doc__ is not None else '' for func in (t1, t2, t3, t4, t5, TC): - self.assertEqual(signature(func), func.tip + '\ndoc') + self.assertEqual(signature(func), func.tip + doc) def test_methods(self): + doc = '\ndoc' if TC.__doc__ is not None else '' for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__): - self.assertEqual(signature(meth), meth.tip + "\ndoc") - self.assertEqual(signature(TC.cm), "(a)\ndoc") - self.assertEqual(signature(TC.sm), "(b)\ndoc") + self.assertEqual(signature(meth), meth.tip + doc) + self.assertEqual(signature(TC.cm), "(a)" + doc) + self.assertEqual(signature(TC.sm), "(b)" + doc) def test_bound_methods(self): # test that first parameter is correctly removed from argspec + doc = '\ndoc' if TC.__doc__ is not None else '' for meth, mtip in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"), (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),): - self.assertEqual(signature(meth), mtip + "\ndoc") + self.assertEqual(signature(meth), mtip + doc) def test_starred_parameter(self): # test that starred first parameter is *not* removed from argspec diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py new file mode 100644 --- /dev/null +++ b/lib-python/2.7/idlelib/idle_test/test_io.py @@ -0,0 +1,267 @@ +import unittest +import io +from idlelib.PyShell import PseudoInputFile, PseudoOutputFile +from test import test_support as support + + +class Base(object): + def __str__(self): + return '%s:str' % type(self).__name__ + def __unicode__(self): + return '%s:unicode' % type(self).__name__ + def __len__(self): + return 3 + def __iter__(self): + return iter('abc') + def __getitem__(self, *args): + return '%s:item' % type(self).__name__ + def __getslice__(self, *args): + return '%s:slice' % type(self).__name__ + +class S(Base, str): + pass + +class U(Base, unicode): + pass + +class BA(Base, bytearray): + pass + +class MockShell: + def __init__(self): + self.reset() + + def write(self, *args): + self.written.append(args) + + def readline(self): + return self.lines.pop() + + def close(self): + pass + + def reset(self): + self.written = [] + + def push(self, lines): + self.lines = list(lines)[::-1] + + +class PseudeOutputFilesTest(unittest.TestCase): + def test_misc(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.encoding, 'utf-8') + self.assertIsNone(f.errors) + self.assertIsNone(f.newlines) + self.assertEqual(f.name, '') + self.assertFalse(f.closed) + self.assertTrue(f.isatty()) + self.assertFalse(f.readable()) + self.assertTrue(f.writable()) + self.assertFalse(f.seekable()) + + def test_unsupported(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertRaises(IOError, f.fileno) + self.assertRaises(IOError, f.tell) + self.assertRaises(IOError, f.seek, 0) + self.assertRaises(IOError, f.read, 0) + self.assertRaises(IOError, f.readline, 0) + + def test_write(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + f.write('test') + self.assertEqual(shell.written, [('test', 'stdout')]) + shell.reset() + f.write('t\xe8st') + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + shell.reset() + f.write(u't\xe8st') + self.assertEqual(shell.written, [(u't\xe8st', 'stdout')]) + shell.reset() + + f.write(S('t\xe8st')) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + f.write(BA('t\xe8st')) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + f.write(U(u't\xe8st')) + self.assertEqual(shell.written, [(u't\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), unicode) + shell.reset() + + self.assertRaises(TypeError, f.write) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, 'test', 'spam') + self.assertEqual(shell.written, []) + + def test_writelines(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + f.writelines([]) + self.assertEqual(shell.written, []) + shell.reset() + f.writelines(['one\n', 'two']) + self.assertEqual(shell.written, + [('one\n', 'stdout'), ('two', 'stdout')]) + shell.reset() + f.writelines(['on\xe8\n', 'tw\xf2']) + self.assertEqual(shell.written, + [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')]) + shell.reset() + f.writelines([u'on\xe8\n', u'tw\xf2']) + self.assertEqual(shell.written, + [(u'on\xe8\n', 'stdout'), (u'tw\xf2', 'stdout')]) + shell.reset() + + f.writelines([S('t\xe8st')]) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + f.writelines([BA('t\xe8st')]) + self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + f.writelines([U(u't\xe8st')]) + self.assertEqual(shell.written, [(u't\xe8st', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), unicode) + shell.reset() + + self.assertRaises(TypeError, f.writelines) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, [123]) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.writelines, [], []) + self.assertEqual(shell.written, []) + + def test_close(self): + shell = MockShell() + f = PseudoOutputFile(shell, 'stdout', 'utf-8') + self.assertFalse(f.closed) + f.write('test') + f.close() + self.assertTrue(f.closed) + self.assertRaises(ValueError, f.write, 'x') + self.assertEqual(shell.written, [('test', 'stdout')]) + f.close() + self.assertRaises(TypeError, f.close, 1) + + +class PseudeInputFilesTest(unittest.TestCase): + def test_misc(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + self.assertIsInstance(f, io.TextIOBase) + self.assertEqual(f.encoding, 'utf-8') + self.assertIsNone(f.errors) + self.assertIsNone(f.newlines) + self.assertEqual(f.name, '') + self.assertFalse(f.closed) + self.assertTrue(f.isatty()) + self.assertTrue(f.readable()) + self.assertFalse(f.writable()) + self.assertFalse(f.seekable()) + + def test_unsupported(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + self.assertRaises(IOError, f.fileno) + self.assertRaises(IOError, f.tell) + self.assertRaises(IOError, f.seek, 0) + self.assertRaises(IOError, f.write, 'x') + self.assertRaises(IOError, f.writelines, ['x']) + + def test_read(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(), 'one\ntwo\n') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(-1), 'one\ntwo\n') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.read(None), 'one\ntwo\n') + shell.push(['one\n', 'two\n', 'three\n', '']) + self.assertEqual(f.read(2), 'on') + self.assertEqual(f.read(3), 'e\nt') + self.assertEqual(f.read(10), 'wo\nthree\n') + + shell.push(['one\n', 'two\n']) + self.assertEqual(f.read(0), '') + self.assertRaises(TypeError, f.read, 1.5) + self.assertRaises(TypeError, f.read, '1') + self.assertRaises(TypeError, f.read, 1, 1) + + def test_readline(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', 'three\n', 'four\n']) + self.assertEqual(f.readline(), 'one\n') + self.assertEqual(f.readline(-1), 'two\n') + self.assertEqual(f.readline(None), 'three\n') + shell.push(['one\ntwo\n']) + self.assertEqual(f.readline(), 'one\n') + self.assertEqual(f.readline(), 'two\n') + shell.push(['one', 'two', 'three']) + self.assertEqual(f.readline(), 'one') + self.assertEqual(f.readline(), 'two') + shell.push(['one\n', 'two\n', 'three\n']) + self.assertEqual(f.readline(2), 'on') + self.assertEqual(f.readline(1), 'e') + self.assertEqual(f.readline(1), '\n') + self.assertEqual(f.readline(10), 'two\n') + + shell.push(['one\n', 'two\n']) + self.assertEqual(f.readline(0), '') + self.assertRaises(TypeError, f.readlines, 1.5) + self.assertRaises(TypeError, f.readlines, '1') + self.assertRaises(TypeError, f.readlines, 1, 1) + + def test_readlines(self): + shell = MockShell() + f = PseudoInputFile(shell, 'stdin', 'utf-8') + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(-1), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(None), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(0), ['one\n', 'two\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(3), ['one\n']) + shell.push(['one\n', 'two\n', '']) + self.assertEqual(f.readlines(4), ['one\n', 'two\n']) + + shell.push(['one\n', 'two\n', '']) + self.assertRaises(TypeError, f.readlines, 1.5) + self.assertRaises(TypeError, f.readlines, '1') + self.assertRaises(TypeError, f.readlines, 1, 1) + From noreply at buildbot.pypy.org Tue Aug 11 12:01:00 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 12:01:00 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: added missing vectorize=True for the jit drivers in loop.py Message-ID: <20150811100100.EB5791C11D3@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78881:4dcddb2caee3 Date: 2015-08-11 11:59 +0200 http://bitbucket.org/pypy/pypy/changeset/4dcddb2caee3/ Log: added missing vectorize=True for the jit drivers in loop.py diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -15,7 +15,7 @@ from pypy.module.micronumpy.descriptor import get_dtype_cache from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction, CodeUniqueIds) + UserDelAction) from pypy.interpreter.pyframe import PyFrame @@ -93,7 +93,6 @@ self.config = config self.interned_strings = make_weak_value_dictionary(self, str, W_Root) - self.code_unique_ids = CodeUniqueIds() self.builtin = DictObject({}) self.FrameClass = PyFrame self.threadlocals = ThreadLocals() @@ -165,6 +164,10 @@ lgt = (stop - start - 1) / step + 1 return (start, stop, step, lgt) + def unicode_from_object(self, w_item): + # XXX + return StringObject("") + @specialize.argtype(1) def wrap(self, obj): if isinstance(obj, float): @@ -281,6 +284,12 @@ return w_obj.v raise NotImplementedError + def unicode_w(self, w_obj): + # XXX + if isinstance(w_obj, StringObject): + return unicode(w_obj.v) + raise NotImplementedError + def int(self, w_obj): if isinstance(w_obj, IntObject): return w_obj diff --git a/pypy/module/micronumpy/flatiter.py b/pypy/module/micronumpy/flatiter.py --- a/pypy/module/micronumpy/flatiter.py +++ b/pypy/module/micronumpy/flatiter.py @@ -97,7 +97,7 @@ finally: self.iter.reset(self.state, mutate=True) - def descr___array_wrap__(self, space, obj): + def descr___array_wrap__(self, space, obj, w_context=None): return obj W_FlatIterator.typedef = TypeDef("numpy.flatiter", diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -9,7 +9,7 @@ from pypy.module.micronumpy import support, constants as NPY from pypy.module.micronumpy.base import W_NDimArray, convert_to_array from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \ - AllButAxisIter + AllButAxisIter, ArrayIter from pypy.interpreter.argument import Arguments @@ -202,29 +202,28 @@ reduce_flat_driver = jit.JitDriver( name='numpy_reduce_flat', - greens = ['shapelen', 'func', 'done_func', 'calc_dtype'], reds = 'auto') + greens = ['shapelen', 'func', 'done_func', 'calc_dtype'], reds = 'auto', + vectorize = True) -def compute_reduce(space, obj, calc_dtype, func, done_func, identity): - obj_iter, obj_state = obj.create_iter() +def reduce_flat(space, func, w_arr, calc_dtype, done_func, identity): + obj_iter, obj_state = w_arr.create_iter() if identity is None: cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) obj_state = obj_iter.next(obj_state) else: cur_value = identity.convert_to(space, calc_dtype) - shapelen = len(obj.get_shape()) + shapelen = len(w_arr.get_shape()) while not obj_iter.done(obj_state): - reduce_driver.jit_merge_point(shapelen=shapelen, func=func, - done_func=done_func, - calc_dtype=calc_dtype) + reduce_flat_driver.jit_merge_point( + shapelen=shapelen, func=func, + done_func=done_func, calc_dtype=calc_dtype) rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) if done_func is not None and done_func(calc_dtype, rval): return rval cur_value = func(calc_dtype, cur_value, rval) obj_state = obj_iter.next(obj_state) - rval = None return cur_value - reduce_driver = jit.JitDriver( name='numpy_reduce', greens=['shapelen', 'func', 'dtype'], reds='auto', @@ -316,7 +315,8 @@ accumulate_driver = jit.JitDriver( name='numpy_accumulate', - greens=['shapelen', 'func', 'calc_dtype'], reds='auto', + greens=['shapelen', 'func', 'calc_dtype'], + reds='auto', vectorize=True) @@ -367,7 +367,8 @@ where_driver = jit.JitDriver(name='numpy_where', greens = ['shapelen', 'dtype', 'arr_dtype'], - reds = 'auto', vectorize=True) + reds = 'auto', + vectorize=True) def where(space, out, shape, arr, x, y, dtype): out_iter, out_state = out.create_iter(shape) diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -7,6 +7,7 @@ # structures to describe slicing class BaseChunk(object): + _attrs_ = ['step'] pass diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -276,7 +276,7 @@ loop.accumulate_flat( space, self.func, obj, dtype, out, self.identity) if call__array_wrap__: - out = space.call_method(obj, '__array_wrap__', out) + out = space.call_method(obj, '__array_wrap__', out, None) return out axis_flags = [False] * shapelen @@ -313,7 +313,7 @@ out.implementation.setitem(0, res) res = out if call__array_wrap__: - res = space.call_method(obj, '__array_wrap__', res) + res = space.call_method(obj, '__array_wrap__', res, None) return res else: @@ -360,7 +360,7 @@ loop.reduce( space, self.func, obj, axis_flags, dtype, out, self.identity) if call__array_wrap__: - out = space.call_method(obj, '__array_wrap__', out) + out = space.call_method(obj, '__array_wrap__', out, None) return out def descr_outer(self, space, __args__): From noreply at buildbot.pypy.org Tue Aug 11 12:13:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 12:13:53 +0200 (CEST) Subject: [pypy-commit] pypy default: Use .startswith() here, for the vecopt branch Message-ID: <20150811101353.98BCC1C11D6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78882:701244bf3ebf Date: 2015-08-11 11:14 +0100 http://bitbucket.org/pypy/pypy/changeset/701244bf3ebf/ Log: Use .startswith() here, for the vecopt branch diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -6,7 +6,7 @@ from rpython.rtyper.tool import rffi_platform as platform from rpython.jit.backend import detect_cpu -if detect_cpu.autodetect() != detect_cpu.MODEL_X86_64: +if not detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64): py.test.skip("rvmprof only supports x86-64 CPUs for now") @@ -71,7 +71,7 @@ func.c_name = cont_name func._dont_inline_ = True - assert detect_cpu.autodetect() == detect_cpu.MODEL_X86_64, ( + assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), ( "rvmprof only supports x86-64 CPUs for now") # mapping of argument count (not counting the final uid argument) to From noreply at buildbot.pypy.org Tue Aug 11 12:27:31 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 12:27:31 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: Fix for df216970b57e: we now expect ".intval" to be really an int, never a bool Message-ID: <20150811102731.3B7341C11D8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vecopt-merge Changeset: r78883:9299faf9cad1 Date: 2015-08-11 10:13 +0100 http://bitbucket.org/pypy/pypy/changeset/9299faf9cad1/ Log: Fix for df216970b57e: we now expect ".intval" to be really an int, never a bool diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py --- a/pypy/objspace/std/boolobject.py +++ b/pypy/objspace/std/boolobject.py @@ -13,7 +13,7 @@ class W_BoolObject(W_IntObject): def __init__(self, boolval): - self.intval = not not boolval + self.intval = int(not not boolval) def __nonzero__(self): raise Exception("you cannot do that, you must use space.is_true()") diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py --- a/pypy/objspace/std/test/test_boolobject.py +++ b/pypy/objspace/std/test/test_boolobject.py @@ -4,6 +4,10 @@ self.false = self.space.w_False self.wrap = self.space.wrap + def test_init(self): + assert (self.false.intval, type(self.false.intval)) == (0, int) + assert (self.true.intval, type(self.true.intval)) == (1, int) + def test_repr(self): assert self.space.eq_w(self.space.repr(self.true), self.wrap("True")) assert self.space.eq_w(self.space.repr(self.false), self.wrap("False")) From noreply at buildbot.pypy.org Tue Aug 11 12:27:33 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 12:27:33 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: Backed out changeset 9299faf9cad1 Message-ID: <20150811102733.5A5631C11D8@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78884:18e7a066f96f Date: 2015-08-11 12:24 +0200 http://bitbucket.org/pypy/pypy/changeset/18e7a066f96f/ Log: Backed out changeset 9299faf9cad1 diff --git a/pypy/objspace/std/boolobject.py b/pypy/objspace/std/boolobject.py --- a/pypy/objspace/std/boolobject.py +++ b/pypy/objspace/std/boolobject.py @@ -13,7 +13,7 @@ class W_BoolObject(W_IntObject): def __init__(self, boolval): - self.intval = int(not not boolval) + self.intval = not not boolval def __nonzero__(self): raise Exception("you cannot do that, you must use space.is_true()") diff --git a/pypy/objspace/std/test/test_boolobject.py b/pypy/objspace/std/test/test_boolobject.py --- a/pypy/objspace/std/test/test_boolobject.py +++ b/pypy/objspace/std/test/test_boolobject.py @@ -4,10 +4,6 @@ self.false = self.space.w_False self.wrap = self.space.wrap - def test_init(self): - assert (self.false.intval, type(self.false.intval)) == (0, int) - assert (self.true.intval, type(self.true.intval)) == (1, int) - def test_repr(self): assert self.space.eq_w(self.space.repr(self.true), self.wrap("True")) assert self.space.eq_w(self.space.repr(self.false), self.wrap("False")) From noreply at buildbot.pypy.org Tue Aug 11 12:27:35 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 12:27:35 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: Use .startswith() here, for the vecopt branch Message-ID: <20150811102735.6A65F1C11D8@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: vecopt-merge Changeset: r78885:86e62bea7806 Date: 2015-08-11 11:14 +0100 http://bitbucket.org/pypy/pypy/changeset/86e62bea7806/ Log: Use .startswith() here, for the vecopt branch diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -6,7 +6,7 @@ from rpython.rtyper.tool import rffi_platform as platform from rpython.jit.backend import detect_cpu -if detect_cpu.autodetect() != detect_cpu.MODEL_X86_64: +if not detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64): py.test.skip("rvmprof only supports x86-64 CPUs for now") @@ -71,7 +71,7 @@ func.c_name = cont_name func._dont_inline_ = True - assert detect_cpu.autodetect() == detect_cpu.MODEL_X86_64, ( + assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), ( "rvmprof only supports x86-64 CPUs for now") # mapping of argument count (not counting the final uid argument) to From noreply at buildbot.pypy.org Tue Aug 11 12:42:06 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 12:42:06 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: meeeh, syntax error did not pay attentation to the indentation in this file Message-ID: <20150811104206.46EE71C0206@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78886:69fe04c7e304 Date: 2015-08-11 12:42 +0200 http://bitbucket.org/pypy/pypy/changeset/69fe04c7e304/ Log: meeeh, syntax error did not pay attentation to the indentation in this file diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py --- a/rpython/jit/codewriter/test/test_flatten.py +++ b/rpython/jit/codewriter/test/test_flatten.py @@ -826,8 +826,7 @@ "int_return %i0", ] else: - expected.insert(0, - "residual_call_irf_i $<* fn llong_to_int>, I[], R[], F[%f0], -> %i0") + expected.insert(0, "residual_call_irf_i $<* fn llong_to_int>, I[], R[], F[%f0], -> %i0") expectedstr = '\n'.join(expected) self.encoding_test(f, [rffi.cast(FROM, 42)], expectedstr, transform=True) From noreply at buildbot.pypy.org Tue Aug 11 12:45:26 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 12:45:26 +0200 (CEST) Subject: [pypy-commit] pypy default: (cfbolz, arigo): fix no nul propagation of modulo of strings and non-tuples Message-ID: <20150811104526.02CCD1C0206@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r78887:8700b302673f Date: 2015-08-11 12:45 +0200 http://bitbucket.org/pypy/pypy/changeset/8700b302673f/ Log: (cfbolz, arigo): fix no nul propagation of modulo of strings and non-tuples diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -425,8 +425,9 @@ class __extend__(pairtype(SomeString, SomeObject), pairtype(SomeUnicodeString, SomeObject)): - def mod((s_string, args)): - return s_string.__class__() + def mod((s_string, s_arg)): + assert not isinstance(s_arg, SomeTuple) + return pair(s_string, SomeTuple([s_arg])).mod() class __extend__(pairtype(SomeFloat, SomeFloat)): diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2124,6 +2124,16 @@ assert isinstance(s, annmodel.SomeString) assert s.no_nul + def test_no_nul_mod(self): + def f(x): + s = "%d" % x + return s + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeString) + assert s.no_nul + + def test_mul_str0(self): def f(s): return s*10 From noreply at buildbot.pypy.org Tue Aug 11 12:58:32 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 11 Aug 2015 12:58:32 +0200 (CEST) Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: think some more Message-ID: <20150811105832.7FE451C04C1@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-faster-smallobj-sync Changeset: r1930:fe7f0b6ebe7e Date: 2015-08-11 13:01 +0200 http://bitbucket.org/pypy/stmgc/changeset/fe7f0b6ebe7e/ Log: think some more diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h --- a/c8/stm/smallmalloc.h +++ b/c8/stm/smallmalloc.h @@ -52,15 +52,25 @@ /* lists the memory ranges of uncommitted/overflow objs that need to be flushed to other segments on commit (like - large_overflow_objects) */ + large_overflow_objects). (unsorted, a range never overlaps + pages) */ struct list_s *uncommitted_ranges; - /* List of smallobjs that were unreachable in a major GC. We - really free them only on commit/abort because we would - otherwise need to deal with holes generated in the - uncommitted_ranges during major GCs (and I can't think of - a non-quadratic algorithm to deal with this). */ - struct list_s *delayed_free_objs; + /* with "uncommitted_ranges", we do the following during major GCs: + + for seg in segments: + for r in seg.uncommitted_ranges: + size = obj_size_in_seg(seg, (object_t*)r) + assert(r.size % size == 0) + for(obj = r; obj < r + r.size; r += size): + if obj_dies(obj): + adjust_range(r) // split/shrink + + Optionally, we can also merge ranges, sort the list, etc. + But generally, I hope we don't have too many uncommitted + ranges; and I don't know, if there is another way to get + the size(-class) of the page/objs. + */ }; From noreply at buildbot.pypy.org Tue Aug 11 13:18:53 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 13:18:53 +0200 (CEST) Subject: [pypy-commit] pypy default: Must not use jitdriver.jit_merge_point() twice for the same jitdriver instance! Message-ID: <20150811111853.472821C0683@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78888:c91e9f5f0bcd Date: 2015-08-11 12:18 +0100 http://bitbucket.org/pypy/pypy/changeset/c91e9f5f0bcd/ Log: Must not use jitdriver.jit_merge_point() twice for the same jitdriver instance! diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -416,6 +416,9 @@ arg_driver = jit.JitDriver(name='numpy_' + op_name, greens = ['shapelen', 'dtype'], reds = 'auto') + arg_flat_driver = jit.JitDriver(name='numpy_flat_' + op_name, + greens = ['shapelen', 'dtype'], + reds = 'auto') def argmin_argmax(space, w_arr, w_out, axis): from pypy.module.micronumpy.descriptor import get_dtype_cache @@ -457,7 +460,7 @@ state = iter.next(state) shapelen = len(w_arr.get_shape()) while not iter.done(state): - arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) + arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) w_val = iter.getitem(state) new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) if dtype.itemtype.ne(new_best, cur_best): From noreply at buildbot.pypy.org Tue Aug 11 14:14:42 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 11 Aug 2015 14:14:42 +0200 (CEST) Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: WIP Message-ID: <20150811121443.00BC11C0683@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-faster-smallobj-sync Changeset: r1931:b37387a2af20 Date: 2015-08-11 14:17 +0200 http://bitbucket.org/pypy/stmgc/changeset/b37387a2af20/ Log: WIP diff --git a/c8/TODO b/c8/TODO --- a/c8/TODO +++ b/c8/TODO @@ -23,6 +23,8 @@ - avoid __builtin_frame_address(0) in precisely the performance-critical functions like the interpreter main loop +- some lists in PSEGMENT can only grow and never shrink again + check if it matters --------------------------- DONE: diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -1155,6 +1155,7 @@ assert(list_is_empty(STM_PSEGMENT->modified_old_objects)); assert(list_is_empty(STM_PSEGMENT->large_overflow_objects)); + assert(list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges)); assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery)); assert(list_is_empty(STM_PSEGMENT->young_weakrefs)); assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery)); @@ -1243,6 +1244,7 @@ list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->old_objects_with_cards_set); list_clear(STM_PSEGMENT->large_overflow_objects); + list_clear(STM_PSEGMENT->small_overflow_obj_ranges); if (tl != NULL) timing_event(tl, event); @@ -1273,20 +1275,32 @@ #endif } -static void push_large_overflow_objects_to_other_segments(void) +static void push_overflow_objects_to_other_segments(void) { - if (list_is_empty(STM_PSEGMENT->large_overflow_objects)) - return; + if (!list_is_empty(STM_PSEGMENT->large_overflow_objects)) { + acquire_privatization_lock(STM_SEGMENT->segment_num); + LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, + ({ + assert(!(item->stm_flags & GCFLAG_WB_EXECUTED)); + synchronize_object_enqueue(item); + })); + synchronize_objects_flush(); + release_privatization_lock(STM_SEGMENT->segment_num); + } - /* XXX: also pushes small ones right now */ - acquire_privatization_lock(STM_SEGMENT->segment_num); - LIST_FOREACH_R(STM_PSEGMENT->large_overflow_objects, object_t *, - ({ - assert(!(item->stm_flags & GCFLAG_WB_EXECUTED)); - synchronize_object_enqueue(item); - })); - synchronize_objects_flush(); - release_privatization_lock(STM_SEGMENT->segment_num); + if (!list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges)) { + acquire_privatization_lock(STM_SEGMENT->segment_num); + + struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges; + while (!list_is_empty(lst)) { + ssize_t len = (ssize_t)list_pop_item(lst); + stm_char *start = (stm_char*)list_pop_item(lst); + _synchronize_fragment(start, len); + } + + synchronize_objects_flush(); + release_privatization_lock(STM_SEGMENT->segment_num); + } /* we can as well clear the list here, since the objects are only useful if the commit succeeds. And @@ -1297,6 +1311,7 @@ unknown-to-the-segment/uncommitted things. */ list_clear(STM_PSEGMENT->large_overflow_objects); + list_clear(STM_PSEGMENT->small_overflow_obj_ranges); } @@ -1335,7 +1350,7 @@ s_mutex_unlock(); } - push_large_overflow_objects_to_other_segments(); + push_overflow_objects_to_other_segments(); /* push before validate. otherwise they are reachable too early */ @@ -1517,6 +1532,7 @@ list_clear(pseg->objects_pointing_to_nursery); list_clear(pseg->old_objects_with_cards_set); list_clear(pseg->large_overflow_objects); + list_clear(pseg->small_overflow_obj_ranges); list_clear(pseg->young_weakrefs); #pragma pop_macro("STM_SEGMENT") #pragma pop_macro("STM_PSEGMENT") @@ -1709,7 +1725,6 @@ } - static void synchronize_object_enqueue(object_t *obj) { assert(!_is_young(obj)); @@ -1785,3 +1800,41 @@ DEBUG_EXPECT_SEGFAULT(true); } + + + +static void small_overflow_obj_ranges_add(object_t *obj) +{ + assert(is_small_uniform(obj)); + + ssize_t obj_size = stmcb_size_rounded_up( + (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj)); + OPT_ASSERT(obj_size >= 16); + + struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges; + if (!list_is_empty(lst)) { + /* try to merge with other ranges (XXX: quadratic, problem?) */ + stm_char *obj_start = (stm_char*)obj; + long i; + for (i = lst->count - 2; i >= 0; i -= 2) { + stm_char *start = (stm_char*)lst->items[i]; + ssize_t size = (ssize_t)lst->items[i+1]; + + if (start + size == obj_start) { + /* merge! + Note: we cannot overlap pages by adding to a range + in this way, since the first slot in a smallmalloc + page is always unused. Thus, we never merge the + last obj in a page with the first obj in a successor + page. */ + lst->items[i+1] = size + obj_size; + return; + } + } + } + + /* no merge was found */ + STM_PSEGMENT->small_overflow_obj_ranges = + list_append2(STM_PSEGMENT->small_overflow_obj_ranges, + (uintptr_t)obj, (uintptr_t)obj_size); +} diff --git a/c8/stm/core.h b/c8/stm/core.h --- a/c8/stm/core.h +++ b/c8/stm/core.h @@ -104,10 +104,17 @@ /* list of objects created in the current transaction and that survived at least one minor collection. They need to be synchronized to other segments on commit, but they - do not need to be in the commit log entry. - XXX: for now it also contains small overflow objs */ + do not need to be in the commit log entry. */ struct list_s *large_overflow_objects; + /* lists the memory ranges of uncommitted/overflow objs that + need to be flushed to other segments on commit (like + large_overflow_objects). (unsorted, a range never overlaps + pages) */ + /* XXX: not much different from before. Maybe try a ranges list + per size class. */ + struct list_s *small_overflow_obj_ranges; + uint8_t privatization_lock; // XXX KILL uint8_t safe_point; @@ -303,8 +310,10 @@ static void touch_all_pages_of_obj(object_t *obj, size_t obj_size); +static inline void _synchronize_fragment(stm_char *frag, ssize_t frag_size); static void synchronize_object_enqueue(object_t *obj); static void synchronize_objects_flush(void); +static void small_overflow_obj_ranges_add(object_t *obj); static void _signal_handler(int sig, siginfo_t *siginfo, void *context); static bool _stm_validate(void); diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c --- a/c8/stm/forksupport.c +++ b/c8/stm/forksupport.c @@ -98,6 +98,7 @@ STM_PSEGMENT->safe_point = SP_NO_TRANSACTION; list_clear(STM_PSEGMENT->objects_pointing_to_nursery); list_clear(STM_PSEGMENT->large_overflow_objects); + list_clear(STM_PSEGMENT->small_overflow_obj_ranges); s_mutex_unlock(); return; diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -597,6 +597,9 @@ } } + /* XXX: fix small_overflow_obj_ranges */ + abort(); + /* Remove from 'modified_old_objects' all old hashtables that die */ { lst = pseg->modified_old_objects; diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c --- a/c8/stm/nursery.c +++ b/c8/stm/nursery.c @@ -57,7 +57,7 @@ static object_t *find_existing_shadow(object_t *obj); #define GCWORD_MOVED ((object_t *) -1) #define FLAG_SYNC_LARGE 0x01 - +#define FLAG_SYNC_SMALL 0x02 static void minor_trace_if_young(object_t **pobj) { @@ -96,6 +96,8 @@ dprintf(("has_shadow(%p): %p, sz:%lu\n", obj, nobj, size)); + /* XXX: shadows are always large-mallocd */ + nobj_sync_now = FLAG_SYNC_LARGE; goto copy_large_object; } @@ -106,10 +108,12 @@ /* case 1: object is not small enough. Ask gcpage.c for an allocation via largemalloc. */ nobj = (object_t *)allocate_outside_nursery_large(size); + nobj_sync_now = FLAG_SYNC_LARGE; } else { /* case "small enough" */ nobj = (object_t *)allocate_outside_nursery_small(size); + nobj_sync_now = FLAG_SYNC_SMALL; } //dprintf(("move %p -> %p\n", obj, nobj)); @@ -118,7 +122,7 @@ char *realnobj = REAL_ADDRESS(STM_SEGMENT->segment_base, nobj); memcpy(realnobj, realobj, size); - nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE; + nobj_sync_now |= (uintptr_t)nobj; /* SYNC_LARGE set above */ pforwarded_array[0] = GCWORD_MOVED; pforwarded_array[1] = nobj; @@ -133,6 +137,7 @@ /* a young object outside the nursery */ nobj = obj; tree_delete_item(STM_PSEGMENT->young_outside_nursery, (uintptr_t)nobj); + assert(!is_small_uniform(nobj)); nobj_sync_now = ((uintptr_t)nobj) | FLAG_SYNC_LARGE; } @@ -363,31 +368,42 @@ while (!list_is_empty(lst)) { uintptr_t obj_sync_now = list_pop_item(lst); - object_t *obj = (object_t *)(obj_sync_now & ~FLAG_SYNC_LARGE); + object_t *obj = (object_t *)(obj_sync_now & ~(FLAG_SYNC_LARGE|FLAG_SYNC_SMALL)); assert(!_is_in_nursery(obj)); _collect_now(obj); assert(!(obj->stm_flags & GCFLAG_CARDS_SET)); - if (obj_sync_now & FLAG_SYNC_LARGE) { - /* XXX: SYNC_LARGE even set for small objs right now */ - /* this is a newly allocated obj in this transaction. We must - either synchronize the object to other segments now, or - add the object to large_overflow_objects list */ - struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num); - if (pseg->minor_collect_will_commit_now) { - acquire_privatization_lock(pseg->pub.segment_num); - synchronize_object_enqueue(obj); - release_privatization_lock(pseg->pub.segment_num); - } else { + /* the list could have moved while appending */ + lst = STM_PSEGMENT->objects_pointing_to_nursery; + + if (!(obj_sync_now & (FLAG_SYNC_LARGE|FLAG_SYNC_SMALL))) + continue; + + /* obj is a newly allocated obj in this transaction. We must + either synchronize the object to other segments now, or + add the object to large_overflow_objects list */ + struct stm_priv_segment_info_s *pseg = get_priv_segment(STM_SEGMENT->segment_num); + if (pseg->minor_collect_will_commit_now) { + /* small or large obj are directly synced now */ + acquire_privatization_lock(pseg->pub.segment_num); + synchronize_object_enqueue(obj); + release_privatization_lock(pseg->pub.segment_num); + + if (obj_sync_now & FLAG_SYNC_LARGE) { + _cards_cleared_in_object(pseg, obj, false); + } + } else { + /* remember them in lists to sync on commit */ + if (obj_sync_now & FLAG_SYNC_LARGE) { LIST_APPEND(STM_PSEGMENT->large_overflow_objects, obj); + _cards_cleared_in_object(pseg, obj, false); + } else { // SYNC_SMALL + small_overflow_obj_ranges_add(obj); } - _cards_cleared_in_object(pseg, obj, false); } - /* the list could have moved while appending */ - lst = STM_PSEGMENT->objects_pointing_to_nursery; } /* flush all overflow objects to other segments now */ diff --git a/c8/stm/setup.c b/c8/stm/setup.c --- a/c8/stm/setup.c +++ b/c8/stm/setup.c @@ -101,6 +101,7 @@ pr->pub.segment_base = segment_base; pr->modified_old_objects = list_create(); pr->large_overflow_objects = list_create(); + pr->small_overflow_obj_ranges = list_create(); pr->young_weakrefs = list_create(); pr->old_weakrefs = list_create(); pr->objects_pointing_to_nursery = list_create(); @@ -158,7 +159,9 @@ list_free(pr->old_objects_with_cards_set); list_free(pr->modified_old_objects); assert(list_is_empty(pr->large_overflow_objects)); + assert(list_is_empty(pr->small_overflow_obj_ranges)); list_free(pr->large_overflow_objects); + list_free(pr->small_overflow_obj_ranges); list_free(pr->young_weakrefs); list_free(pr->old_weakrefs); tree_free(pr->young_outside_nursery); diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h --- a/c8/stm/smallmalloc.h +++ b/c8/stm/smallmalloc.h @@ -50,12 +50,6 @@ struct small_malloc_data_s { struct small_free_loc_s *loc_free[GC_N_SMALL_REQUESTS]; - /* lists the memory ranges of uncommitted/overflow objs that - need to be flushed to other segments on commit (like - large_overflow_objects). (unsorted, a range never overlaps - pages) */ - struct list_s *uncommitted_ranges; - /* with "uncommitted_ranges", we do the following during major GCs: for seg in segments: From noreply at buildbot.pypy.org Tue Aug 11 14:35:20 2015 From: noreply at buildbot.pypy.org (Raemi) Date: Tue, 11 Aug 2015 14:35:20 +0200 (CEST) Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: implement major collection fixup; tests pass Message-ID: <20150811123520.877E81C089E@cobra.cs.uni-duesseldorf.de> Author: Remi Meier Branch: c8-faster-smallobj-sync Changeset: r1932:bd7979af1bf7 Date: 2015-08-11 14:38 +0200 http://bitbucket.org/pypy/stmgc/changeset/bd7979af1bf7/ Log: implement major collection fixup; tests pass diff --git a/c8/stm/core.c b/c8/stm/core.c --- a/c8/stm/core.c +++ b/c8/stm/core.c @@ -1835,6 +1835,5 @@ /* no merge was found */ STM_PSEGMENT->small_overflow_obj_ranges = - list_append2(STM_PSEGMENT->small_overflow_obj_ranges, - (uintptr_t)obj, (uintptr_t)obj_size); + list_append2(lst, (uintptr_t)obj, (uintptr_t)obj_size); } diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c --- a/c8/stm/gcpage.c +++ b/c8/stm/gcpage.c @@ -597,8 +597,44 @@ } } - /* XXX: fix small_overflow_obj_ranges */ - abort(); + /* fix ranges in small_overflow_obj_ranges that may have + holes from dying objs */ + lst = pseg->small_overflow_obj_ranges; + if (!list_is_empty(lst)) { + long j; + struct list_s *new_lst = list_create(); + for (j = lst->count - 2; j >= 0; j -= 2) { + stm_char *start = (stm_char*)lst->items[j]; + ssize_t size = (ssize_t)lst->items[j+1]; + + struct object_s *realobj = (struct object_s *) + REAL_ADDRESS(pseg->pub.segment_base, start); + size_t obj_size = stmcb_size_rounded_up(realobj); + assert(size % obj_size == 0); + + stm_char *obj = start, *range_end = start + size; + for (; obj < range_end; obj += obj_size) { + /* note: all these obj-slots *are* in use by overflow objs */ + if (!mark_visited_test((object_t*)obj)) { + if (obj != start) { + new_lst = list_append2(new_lst, + (uintptr_t)start, + (uintptr_t)(obj - start)); + } + start = obj + obj_size; // next start + } + } + if (obj != start) { + new_lst = list_append2(new_lst, + (uintptr_t)start, + (uintptr_t)(obj - start)); + } + } + + list_free(lst); + pseg->small_overflow_obj_ranges = new_lst; + } + /* Remove from 'modified_old_objects' all old hashtables that die */ { diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h --- a/c8/stm/smallmalloc.h +++ b/c8/stm/smallmalloc.h @@ -49,22 +49,6 @@ */ struct small_malloc_data_s { struct small_free_loc_s *loc_free[GC_N_SMALL_REQUESTS]; - - /* with "uncommitted_ranges", we do the following during major GCs: - - for seg in segments: - for r in seg.uncommitted_ranges: - size = obj_size_in_seg(seg, (object_t*)r) - assert(r.size % size == 0) - for(obj = r; obj < r + r.size; r += size): - if obj_dies(obj): - adjust_range(r) // split/shrink - - Optionally, we can also merge ranges, sort the list, etc. - But generally, I hope we don't have too many uncommitted - ranges; and I don't know, if there is another way to get - the size(-class) of the page/objs. - */ }; From noreply at buildbot.pypy.org Tue Aug 11 14:37:56 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 14:37:56 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: added out_dim to the list of attributes Message-ID: <20150811123756.8C4BE1C089E@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78889:9b128f66714b Date: 2015-08-11 14:38 +0200 http://bitbucket.org/pypy/pypy/changeset/9b128f66714b/ Log: added out_dim to the list of attributes diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -7,7 +7,7 @@ # structures to describe slicing class BaseChunk(object): - _attrs_ = ['step'] + _attrs_ = ['step','out_dim'] pass From noreply at buildbot.pypy.org Tue Aug 11 14:48:08 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 14:48:08 +0200 (CEST) Subject: [pypy-commit] pypy default: Allow compilation on older Linux systems Message-ID: <20150811124808.CD41D1C04C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78890:7f61ec8c3fd6 Date: 2015-08-11 13:50 +0100 http://bitbucket.org/pypy/pypy/changeset/7f61ec8c3fd6/ Log: Allow compilation on older Linux systems diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -442,19 +442,26 @@ eci = ExternalCompilationInfo(includes=['string.h', 'assert.h', 'sys/prctl.h'], post_include_bits=[""" +/* If we have an old Linux kernel (or compile with old system headers), + the following two macros are not defined. But we would still like + a pypy translated on such a system to run on a more modern system. */ +#ifndef PR_SET_PTRACER +# define PR_SET_PTRACER 0x59616d61 +#endif +#ifndef PR_SET_PTRACER_ANY +# define PR_SET_PTRACER_ANY ((unsigned long)-1) +#endif static void pypy__allow_attach(void) { prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY); - return; } """]) + allow_attach = rffi.llexternal( + "pypy__allow_attach", [], lltype.Void, + compilation_info=eci, _nowrapper=True) else: # Do nothing, there's no prctl - eci = ExternalCompilationInfo(post_include_bits=[ - "static void pypy__allow_attach(void) { return; }"]) - - allow_attach = rffi.llexternal( - "pypy__allow_attach", [], lltype.Void, - compilation_info=eci, _nowrapper=True) + def allow_attach(): + pass def impl_attach_gdb(): import os From noreply at buildbot.pypy.org Tue Aug 11 14:55:02 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 14:55:02 +0200 (CEST) Subject: [pypy-commit] pypy default: (cfbolz, arigo) Improve the error message in the case where the same jitdriver Message-ID: <20150811125502.91FD11C089E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78891:7a82bbfa8b25 Date: 2015-08-11 13:56 +0100 http://bitbucket.org/pypy/pypy/changeset/7a82bbfa8b25/ Log: (cfbolz, arigo) Improve the error message in the case where the same jitdriver is used twice (this is about the case of reds='auto' only) diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -79,6 +79,9 @@ assert methname == 'jit_merge_point', ( "reds='auto' is supported only for jit drivers which " "calls only jit_merge_point. Found a call to %s" % methname) + if jitdriver.numreds is not None: + raise AssertionError("there are multiple jit_merge_points " + "with the same jitdriver") # # compute the set of live variables across the jit_marker alive_v = set() @@ -96,10 +99,7 @@ v.concretetype is not lltype.Void] reds_v = sort_vars(reds_v) op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + jitdriver.numreds = len(reds_v) def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -558,6 +558,22 @@ assert res == 7 - 3 self.check_trace_count(2) + def test_jitdriver_single_jit_merge_point(self): + jitdriver = JitDriver(greens=[], reds='auto') + def g1(n): + jitdriver.jit_merge_point() + return n + def g2(): + jitdriver.jit_merge_point() + def f(n): + if n: + g1(n) + else: + g2() + e = py.test.raises(AssertionError, self.meta_interp, f, [42]) + assert str(e.value) == ("there are multiple jit_merge_points " + "with the same jitdriver") + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU From noreply at buildbot.pypy.org Tue Aug 11 15:23:48 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 15:23:48 +0200 (CEST) Subject: [pypy-commit] pypy default: (arigo, cfbolz): make import_from_mixin deal with _immutable_fields_ Message-ID: <20150811132348.8774A1C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r78892:dd5e385a1000 Date: 2015-08-11 15:23 +0200 http://bitbucket.org/pypy/pypy/changeset/dd5e385a1000/ Log: (arigo, cfbolz): make import_from_mixin deal with _immutable_fields_ diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -806,10 +806,14 @@ flatten = {} caller = sys._getframe(1) caller_name = caller.f_globals.get('__name__') + immutable_fields = [] for base in inspect.getmro(M): if base is object: continue for key, value in base.__dict__.items(): + if key == '_immutable_fields_': + immutable_fields.extend(value) + continue if key.startswith('__') and key.endswith('__'): if key not in special_methods: continue @@ -836,3 +840,5 @@ raise Exception("import_from_mixin: would overwrite the value " "already defined locally for %r" % (key,)) target[key] = value + if immutable_fields: + target['_immutable_fields_'] = target.get('_immutable_fields_', []) + immutable_fields diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -675,3 +675,44 @@ import_from_mixin(M) assert A.f is not M.f assert A.f.__module__ != M.f.__module__ + + +def test_import_from_mixin_immutable_fields(): + class A(object): + _immutable_fields_ = ['a'] + + class B(object): + _immutable_fields_ = ['b'] + import_from_mixin(A) + + assert B._immutable_fields_ == ['b', 'a'] + assert A._immutable_fields_ == ['a'] + + + class B(object): + import_from_mixin(A) + + assert B._immutable_fields_ == ['a'] + + class C(A): + _immutable_fields_ = ['c'] + + + class B(object): + import_from_mixin(C) + + assert B._immutable_fields_ == ['c', 'a'] + + class B(object): + _immutable_fields_ = ['b'] + import_from_mixin(C) + + assert B._immutable_fields_ == ['b', 'c', 'a'] + + + class B(object): + _immutable_fields_ = ['b'] + class BA(B): + import_from_mixin(C) + + assert BA._immutable_fields_ == ['c', 'a'] From noreply at buildbot.pypy.org Tue Aug 11 15:25:47 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 15:25:47 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: merge default Message-ID: <20150811132547.8C20A1C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78893:d3a646fcaace Date: 2015-08-11 15:25 +0200 http://bitbucket.org/pypy/pypy/changeset/d3a646fcaace/ Log: merge default diff too long, truncating to 2000 out of 5220 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -55,3 +55,11 @@ .. branch: nditer-revisited Implement nditer 'buffered' flag and fix some edge cases + +.. branch: ufunc-reduce + +Allow multiple axes in ufunc.reduce() + +.. branch: fix-tinylang-goals + +Update tinylang goals to match current rpython diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -11,7 +11,7 @@ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction, CodeUniqueIds) + UserDelAction) from pypy.interpreter.error import OperationError, new_exception_class, oefmt from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary @@ -391,7 +391,6 @@ self.actionflag = ActionFlag() # changed by the signal module self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) - self.code_unique_ids = CodeUniqueIds() self._code_of_sys_exc_info = None # can be overridden to a subclass @@ -670,16 +669,6 @@ assert ec is not None return ec - def register_code_callback(self, callback): - cui = self.code_unique_ids - cui.code_callback = callback - - def register_code_object(self, pycode): - cui = self.code_unique_ids - if cui.code_callback is None: - return - cui.code_callback(self, pycode) - def _freeze_(self): return True diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -590,11 +590,3 @@ # there is no list of length n: if n is large, then the GC # will run several times while walking the list, but it will # see lower and lower memory usage, with no lower bound of n. - -class CodeUniqueIds(object): - def __init__(self): - if sys.maxint == 2147483647: - self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit - else: - self.code_unique_id = 0x7000000000000000 - self.code_callback = None diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -85,7 +85,7 @@ self.magic = magic self._signature = cpython_code_signature(self) self._initialize() - space.register_code_object(self) + self._init_ready() self.vprof = valueprof.ValueProf(self.co_nlocals) self.printed = [False] * self.co_nlocals @@ -129,14 +129,8 @@ from pypy.objspace.std.mapdict import init_mapdict_cache init_mapdict_cache(self) - cui = self.space.code_unique_ids - self._unique_id = cui.code_unique_id - cui.code_unique_id += 4 # so we have two bits that we can mark stuff - # with - - def _get_full_name(self): - return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno, - self.co_filename) + def _init_ready(self): + "This is a hook for the vmprof module, which overrides this method." def _cleanup_(self): if (self.magic == cpython_magic and diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -62,6 +62,7 @@ } interpleveldefs = { + 'attach_gdb' : 'interp_magic.attach_gdb', 'internal_repr' : 'interp_magic.internal_repr', 'bytebuffer' : 'bytebuffer.bytebuffer', 'identity_dict' : 'interp_identitydict.W_IdentityDict', @@ -100,8 +101,6 @@ def setup_after_space_initialization(self): """NOT_RPYTHON""" - if not self.space.config.translating: - self.extra_interpdef('interp_pdb', 'interp_magic.interp_pdb') if self.space.config.objspace.std.withmethodcachecounter: self.extra_interpdef('method_cache_counter', 'interp_magic.method_cache_counter') diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -15,12 +15,10 @@ return space.wrap('%r' % (w_object,)) -def interp_pdb(space): - """Run an interp-level pdb. - This is not available in translated versions of PyPy.""" - assert not we_are_translated() - import pdb - pdb.set_trace() +def attach_gdb(space): + """Run an interp-level gdb (or pdb when untranslated)""" + from rpython.rlib.debug import attach_gdb + attach_gdb() @unwrap_spec(name=str) diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -209,7 +209,7 @@ # EAGAIN after already some data was received, return it. # Note that we can get EAGAIN while there is buffered data # waiting; read that too. - if is_wouldblock_error(e): + if is_wouldblock_error(e.errno): m = stream.count_buffered_bytes() if m > 0: result.append(stream.read(min(n, m))) @@ -321,6 +321,10 @@ self.getstream() # check if the file is still open return os.isatty(self.fd) + def direct_readinto(self, w_rwbuffer): + from pypy.module._file.readinto import direct_readinto + return direct_readinto(self, w_rwbuffer) + # ____________________________________________________________ # # The 'file_' methods are the one exposed to app-level. @@ -413,6 +417,9 @@ Notice that when in non-blocking mode, less data than what was requested may be returned, even if no size parameter was given.""") + _decl(locals(), "readinto", + """readinto(buf) -> length. Read into the given read-write buffer.""") + _decl(locals(), "readline", """readline([size]) -> next line from the file, as a string. @@ -508,16 +515,6 @@ for w_line in lines: self.file_write(w_line) - def file_readinto(self, w_rwbuffer): - """readinto() -> Undocumented. Don't use this; it may go away.""" - # XXX not the most efficient solution as it doesn't avoid the copying - space = self.space - rwbuffer = space.writebuf_w(w_rwbuffer) - w_data = self.file_read(rwbuffer.getlength()) - data = space.str_w(w_data) - rwbuffer.setslice(0, data) - return space.wrap(len(data)) - # ____________________________________________________________ @@ -603,7 +600,6 @@ cls=W_File, doc="Support for 'print'."), __repr__ = interp2app(W_File.file__repr__), - readinto = interp2app(W_File.file_readinto), writelines = interp2app(W_File.file_writelines), __exit__ = interp2app(W_File.file__exit__), __weakref__ = make_weakref_descr(W_File), @@ -632,10 +628,10 @@ MAYBE_EAGAIN = getattr(errno, 'EAGAIN', None) MAYBE_EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', None) -def is_wouldblock_error(e): - if MAYBE_EAGAIN is not None and e.errno == MAYBE_EAGAIN: +def is_wouldblock_error(errno): + if MAYBE_EAGAIN is not None and errno == MAYBE_EAGAIN: return True - if MAYBE_EWOULDBLOCK is not None and e.errno == MAYBE_EWOULDBLOCK: + if MAYBE_EWOULDBLOCK is not None and errno == MAYBE_EWOULDBLOCK: return True return False diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py new file mode 100644 --- /dev/null +++ b/pypy/module/_file/readinto.py @@ -0,0 +1,81 @@ +import sys, errno +from rpython.rlib import rposix +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.module._file.interp_file import is_wouldblock_error, signal_checker + +_WIN32 = sys.platform.startswith('win') +UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' + +os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) + + +def direct_readinto(self, w_rwbuffer): + rwbuffer = self.space.writebuf_w(w_rwbuffer) + stream = self.getstream() + size = rwbuffer.getlength() + target_address = lltype.nullptr(rffi.CCHARP.TO) + fd = -1 + target_pos = 0 + + if size > 64: + try: + target_address = rwbuffer.get_raw_address() + except ValueError: + pass + else: + fd = stream.try_to_find_file_descriptor() + + if fd < 0 or not target_address: + # fall-back + MAX_PART = 1024 * 1024 # 1 MB + while size > MAX_PART: + data = self.direct_read(MAX_PART) + rwbuffer.setslice(target_pos, data) + target_pos += len(data) + size -= len(data) + if len(data) != MAX_PART: + break + else: + data = self.direct_read(size) + rwbuffer.setslice(target_pos, data) + target_pos += len(data) + + else: + # optimized case: reading more than 64 bytes into a rwbuffer + # with a valid raw address + self.check_readable() + + # first "read" the part that is already sitting in buffers, if any + initial_size = min(size, stream.count_buffered_bytes()) + if initial_size > 0: + data = stream.read(initial_size) + rwbuffer.setslice(target_pos, data) + target_pos += len(data) + size -= len(data) + + # then call os_read() to get the rest + if size > 0: + stream.flush() + while True: + got = os_read(fd, rffi.ptradd(target_address, target_pos), size) + if got > 0: + target_pos += got + size -= got + if size <= 0: + break + elif got == 0: + break + else: + err = rposix.get_saved_errno() + if err == errno.EINTR: + signal_checker(self.space)() + continue + if is_wouldblock_error(err) and target_pos > 0: + break + raise OSError(err, "read error") + keepalive_until_here(rwbuffer) + + return self.space.wrap(target_pos) diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -2,7 +2,7 @@ class Module(MixedModule): """ - Write me :) + VMProf for PyPy: a statistical profiler """ appleveldefs = { } @@ -10,9 +10,13 @@ interpleveldefs = { 'enable': 'interp_vmprof.enable', 'disable': 'interp_vmprof.disable', + 'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError', } - def setup_after_space_initialization(self): - # force the __extend__ hacks to occur early - from pypy.module._vmprof.interp_vmprof import VMProf - self.vmprof = VMProf() + +# Force the __extend__ hacks and method replacements to occur +# early. Without this, for example, 'PyCode._init_ready' was +# already found by the annotator to be the original empty +# method, and the annotator doesn't notice that interp_vmprof.py +# (loaded later) replaces this method. +import pypy.module._vmprof.interp_vmprof diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -1,252 +1,74 @@ -import py, os, sys -from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance -from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib import jit, rposix, rgc -from rpython.rlib.rarithmetic import ovfcheck_float_to_int -from rpython.rtyper.tool import rffi_platform as platform -from rpython.rlib.rstring import StringBuilder -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import oefmt, wrap_oserror, OperationError +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pycode import PyCode +from pypy.interpreter.baseobjspace import W_Root +from rpython.rlib import rvmprof -ROOT = py.path.local(__file__).join('..') -SRC = ROOT.join('src') +# ____________________________________________________________ -# by default, we statically link vmprof.c into pypy; however, if you set -# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so -# which is expected to be inside pypy/module/_vmprof/src: this is very useful -# during development. Note that you have to manually build libvmprof by -# running make inside the src dir -DYNAMIC_VMPROF = False -if sys.platform.startswith('linux'): - libs = ['dl'] -else: - libs = [] +_get_code = lambda frame, w_inputvalue, operr: frame.pycode +_decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root) +my_execute_frame = _decorator(PyFrame.execute_frame) -eci_kwds = dict( - include_dirs = [SRC], - includes = ['vmprof.h', 'trampoline.h'], - separate_module_files = [SRC.join('trampoline.vmprof.s')], - libraries = libs, - - post_include_bits=[""" - int pypy_vmprof_init(void); - """], - - separate_module_sources=[""" - int pypy_vmprof_init(void) { - return vmprof_set_mainloop(pypy_execute_frame_trampoline, 0, - NULL); - } - """], - ) - - -if DYNAMIC_VMPROF: - eci_kwds['libraries'] += ['vmprof'] - eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC] -else: - eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')] - -eci = ExternalCompilationInfo(**eci_kwds) - -check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[ - SRC.join('fake_pypy_api.c')])) - -platform.verify_eci(check_eci) - -pypy_execute_frame_trampoline = rffi.llexternal( - "pypy_execute_frame_trampoline", - [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed], - llmemory.GCREF, - compilation_info=eci, - _nowrapper=True, sandboxsafe=True, - random_effects_on_gcobjs=True) - -pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], rffi.INT, - compilation_info=eci) -vmprof_enable = rffi.llexternal("vmprof_enable", - [rffi.INT, rffi.LONG, rffi.INT, - rffi.CCHARP, rffi.INT], - rffi.INT, compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) -vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, - compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) -vmprof_get_error = rffi.llexternal("vmprof_get_error", [], rffi.CCHARP, - compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) - -vmprof_register_virtual_function = rffi.llexternal( - "vmprof_register_virtual_function", - [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, - compilation_info=eci, _nowrapper=True) - -original_execute_frame = PyFrame.execute_frame.im_func -original_execute_frame.c_name = 'pypy_pyframe_execute_frame' -original_execute_frame._dont_inline_ = True class __extend__(PyFrame): - def execute_frame(frame, w_inputvalue=None, operr=None): - # go through the asm trampoline ONLY if we are translated but not being JITted. - # - # If we are not translated, we obviously don't want to go through the - # trampoline because there is no C function it can call. - # - # If we are being JITted, we want to skip the trampoline, else the JIT - # cannot see throug it - if we_are_translated() and not jit.we_are_jitted(): - # if we are translated, call the trampoline - gc_frame = cast_instance_to_gcref(frame) - gc_inputvalue = cast_instance_to_gcref(w_inputvalue) - gc_operr = cast_instance_to_gcref(operr) - unique_id = frame.pycode._unique_id - gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, - gc_operr, unique_id) - return cast_base_ptr_to_instance(W_Root, gc_result) - else: - return original_execute_frame(frame, w_inputvalue, operr) + def execute_frame(self, w_inputvalue=None, operr=None): + # indirection for the optional arguments + return my_execute_frame(self, w_inputvalue, operr) +def _safe(s): + if len(s) > 110: + s = s[:107] + '...' + return s.replace(':', ';') -def write_long_to_string_builder(l, b): - if sys.maxint == 2147483647: - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) - else: - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) - b.append(chr((l >> 32) & 0xff)) - b.append(chr((l >> 40) & 0xff)) - b.append(chr((l >> 48) & 0xff)) - b.append(chr((l >> 56) & 0xff)) +def _get_full_name(pycode): + # careful, must not have extraneous ':' or be longer than 255 chars + return "py:%s:%d:%s" % (_safe(pycode.co_name), pycode.co_firstlineno, + _safe(pycode.co_filename)) -def try_cast_to_pycode(gcref): - return rgc.try_cast_gcref_to_instance(PyCode, gcref) +rvmprof.register_code_object_class(PyCode, _get_full_name) -MAX_CODES = 1000 -class VMProf(object): - def __init__(self): - self.is_enabled = False - self.ever_enabled = False - self.fileno = -1 - self.current_codes = [] +def _init_ready(pycode): + rvmprof.register_code(pycode, _get_full_name) - def enable(self, space, fileno, period_usec): - if self.is_enabled: - raise oefmt(space.w_ValueError, "_vmprof already enabled") - self.fileno = fileno - self.is_enabled = True - self.write_header(fileno, period_usec) - if not self.ever_enabled: - if we_are_translated(): - res = pypy_vmprof_init() - if res: - raise OperationError( - space.w_IOError, - space.wrap(rffi.charp2str(vmprof_get_error()))) - self.ever_enabled = True - self.gather_all_code_objs(space) - space.register_code_callback(vmprof_register_code) - if we_are_translated(): - # does not work untranslated - res = vmprof_enable(fileno, period_usec, 0, - lltype.nullptr(rffi.CCHARP.TO), 0) - else: - res = 0 - if res == -1: - raise wrap_oserror(space, OSError(rposix.get_saved_errno(), - "_vmprof.enable")) +PyCode._init_ready = _init_ready - def gather_all_code_objs(self, space): - all_code_objs = rgc.do_get_objects(try_cast_to_pycode) - for code in all_code_objs: - self.register_code(space, code) - def write_header(self, fileno, period_usec): - assert period_usec > 0 - b = StringBuilder() - write_long_to_string_builder(0, b) - write_long_to_string_builder(3, b) - write_long_to_string_builder(0, b) - write_long_to_string_builder(period_usec, b) - write_long_to_string_builder(0, b) - b.append('\x04') # interp name - b.append(chr(len('pypy'))) - b.append('pypy') - os.write(fileno, b.build()) +# ____________________________________________________________ - def register_code(self, space, code): - if self.fileno == -1: - raise OperationError(space.w_RuntimeError, - space.wrap("vmprof not running")) - self.current_codes.append(code) - if len(self.current_codes) >= MAX_CODES: - self._flush_codes(space) - def _flush_codes(self, space): - b = StringBuilder() - for code in self.current_codes: - name = code._get_full_name() - b.append('\x02') - write_long_to_string_builder(code._unique_id, b) - write_long_to_string_builder(len(name), b) - b.append(name) - os.write(self.fileno, b.build()) - self.current_codes = [] +class Cache: + def __init__(self, space): + self.w_VMProfError = space.new_exception_class("_vmprof.VMProfError") - def disable(self, space): - if not self.is_enabled: - raise oefmt(space.w_ValueError, "_vmprof not enabled") - self.is_enabled = False - space.register_code_callback(None) - self._flush_codes(space) - self.fileno = -1 - if we_are_translated(): - # does not work untranslated - res = vmprof_disable() - else: - res = 0 - if res == -1: - raise wrap_oserror(space, OSError(rposix.get_saved_errno(), - "_vmprof.disable")) +def VMProfError(space, e): + w_VMProfError = space.fromcache(Cache).w_VMProfError + return OperationError(w_VMProfError, space.wrap(e.msg)) -def vmprof_register_code(space, code): - from pypy.module._vmprof import Module - mod_vmprof = space.getbuiltinmodule('_vmprof') - assert isinstance(mod_vmprof, Module) - mod_vmprof.vmprof.register_code(space, code) @unwrap_spec(fileno=int, period=float) -def enable(space, fileno, period=0.01): # default 100 Hz - from pypy.module._vmprof import Module - mod_vmprof = space.getbuiltinmodule('_vmprof') - assert isinstance(mod_vmprof, Module) - # +def enable(space, fileno, period): + """Enable vmprof. Writes go to the given 'fileno', a file descriptor + opened for writing. *The file descriptor must remain open at least + until disable() is called.* + + 'interval' is a float representing the sampling interval, in seconds. + Must be smaller than 1.0 + """ try: - period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) - if period_usec <= 0 or period_usec >= 1e6: - # we don't want seconds here at all - raise ValueError - except (ValueError, OverflowError): - raise OperationError(space.w_ValueError, - space.wrap("'period' too large or non positive")) - # - mod_vmprof.vmprof.enable(space, fileno, period_usec) + rvmprof.enable(fileno, period) + except rvmprof.VMProfError, e: + raise VMProfError(space, e) def disable(space): - from pypy.module._vmprof import Module - mod_vmprof = space.getbuiltinmodule('_vmprof') - assert isinstance(mod_vmprof, Module) - mod_vmprof.vmprof.disable(space) - + """Disable vmprof. Remember to close the file descriptor afterwards + if necessary. + """ + try: + rvmprof.disable() + except rvmprof.VMProfError, e: + raise VMProfError(space, e) diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h deleted file mode 100644 --- a/pypy/module/_vmprof/src/config.h +++ /dev/null @@ -1,6 +0,0 @@ -#define HAVE_SYS_UCONTEXT_H -#if defined(__FreeBSD__) || defined(__APPLE__) -#define PC_FROM_UCONTEXT uc_mcontext.mc_rip -#else -#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP] -#endif diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c deleted file mode 100644 --- a/pypy/module/_vmprof/src/fake_pypy_api.c +++ /dev/null @@ -1,4 +0,0 @@ - -void pypy_pyframe_execute_frame(void) -{ -} diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c deleted file mode 100644 --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ /dev/null @@ -1,80 +0,0 @@ - -#ifdef PYPY_JIT_CODEMAP - -extern volatile int pypy_codemap_currently_invalid; - -void *pypy_find_codemap_at_addr(long addr, long *start_addr); -long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, - long *current_pos_addr); -long pypy_jit_stack_depth_at_loc(long loc); - -#endif - - -void vmprof_set_tramp_range(void* start, void* end) -{ -} - -int custom_sanity_check() -{ -#ifdef PYPY_JIT_CODEMAP - return !pypy_codemap_currently_invalid; -#else - return 1; -#endif -} - -static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) { -#ifdef PYPY_JIT_CODEMAP - intptr_t ip_l = (intptr_t)ip; - return pypy_jit_stack_depth_at_loc(ip_l); -#else - return 0; -#endif -} - -static long vmprof_write_header_for_jit_addr(void **result, long n, - void *ip, int max_depth) -{ -#ifdef PYPY_JIT_CODEMAP - void *codemap; - long current_pos = 0; - intptr_t id; - long start_addr = 0; - intptr_t addr = (intptr_t)ip; - int start, k; - void *tmp; - - codemap = pypy_find_codemap_at_addr(addr, &start_addr); - if (codemap == NULL) - // not a jit code at all - return n; - - // modify the last entry to point to start address and not the random one - // in the middle - result[n - 1] = (void*)start_addr; - result[n] = (void*)2; - n++; - start = n; - while (n < max_depth) { - id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); - if (id == -1) - // finish - break; - if (id == 0) - continue; // not main codemap - result[n++] = (void *)id; - } - k = 0; - while (k < (n - start) / 2) { - tmp = result[start + k]; - result[start + k] = result[n - k - 1]; - result[n - k - 1] = tmp; - k++; - } - if (n < max_depth) { - result[n++] = (void*)3; - } -#endif - return n; -} diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h deleted file mode 100644 --- a/pypy/module/_vmprof/src/trampoline.h +++ /dev/null @@ -1,1 +0,0 @@ -void* pypy_execute_frame_trampoline(void*, void*, void*, long); diff --git a/pypy/module/_vmprof/src/trampoline.vmprof.s b/pypy/module/_vmprof/src/trampoline.vmprof.s deleted file mode 100644 --- a/pypy/module/_vmprof/src/trampoline.vmprof.s +++ /dev/null @@ -1,15 +0,0 @@ -// NOTE: you need to use TABs, not spaces! - - .text - .globl pypy_execute_frame_trampoline - .type pypy_execute_frame_trampoline, @function -pypy_execute_frame_trampoline: - .cfi_startproc - pushq %rcx - .cfi_def_cfa_offset 16 - call pypy_pyframe_execute_frame at PLT - popq %rcx - .cfi_def_cfa_offset 8 - ret - .cfi_endproc - .size pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c deleted file mode 100644 --- a/pypy/module/_vmprof/src/vmprof.c +++ /dev/null @@ -1,463 +0,0 @@ -/* VMPROF - * - * statistical sampling profiler specifically designed to profile programs - * which run on a Virtual Machine and/or bytecode interpreter, such as Python, - * etc. - * - * The logic to dump the C stack traces is partly stolen from the code in gperftools. - * The file "getpc.h" has been entirely copied from gperftools. - * - * Tested only on gcc, linux, x86_64. - * - * Copyright (C) 2014-2015 - * Antonio Cuni - anto.cuni at gmail.com - * Maciej Fijalkowski - fijall at gmail.com - * - */ - - -#include "getpc.h" // should be first to get the _GNU_SOURCE dfn -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -//#define UNW_LOCAL_ONLY -//#include - -#include "vmprof.h" -#if defined(__FreeBSD__) || defined(__APPLE__) -#define sighandler_t sig_t -#endif - -#define _unused(x) ((void)x) - -#define MAX_FUNC_NAME 128 -#define MAX_STACK_DEPTH 1024 -#define BUFFER_SIZE 8192 - - -static int profile_file = 0; -static char profile_write_buffer[BUFFER_SIZE]; -static int profile_buffer_position = 0; -void* vmprof_mainloop_func; -char* vmprof_error = NULL; -static ptrdiff_t mainloop_sp_offset; -static vmprof_get_virtual_ip_t mainloop_get_virtual_ip; -static long last_period_usec = 0; -static int atfork_hook_installed = 0; - - -/* ************************************************************* - * functions to write a profile file compatible with gperftools - * ************************************************************* - */ - -#define MARKER_STACKTRACE '\x01' -#define MARKER_VIRTUAL_IP '\x02' -#define MARKER_TRAILER '\x03' - -int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL; -int (*unw_step)(unw_cursor_t*) = NULL; -int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; -int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; - -static void prof_word(long x) { - ((long*)(profile_write_buffer + profile_buffer_position))[0] = x; - profile_buffer_position += sizeof(long); -} - -static void prof_header(long period_usec) { - // XXX never used here? - prof_word(0); - prof_word(3); - prof_word(0); - prof_word(period_usec); - prof_word(0); - write(profile_file, profile_write_buffer, profile_buffer_position); - profile_buffer_position = 0; -} - -static void prof_write_stacktrace(void** stack, int depth, int count) { - int i; - char marker = MARKER_STACKTRACE; - - profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE; - prof_word(count); - prof_word(depth); - for(i=0; isp = bp; - bp -= sizeof(void*); - cp2->ip = ((void**)bp)[0]; - // the ret is on the top of the stack minus WORD - return 1; - } -} - - -/* ************************************************************* - * functions to dump the stack trace - * ************************************************************* - */ - -// The original code here has a comment, "stolen from pprof", -// about a "__thread int recursive". But general __thread -// variables are not really supposed to be accessed from a -// signal handler. Moreover, we are using SIGPROF, which -// should not be recursively called on the same thread. -//static __thread int recursive; - -int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) { - void *ip; - int n = 0; - unw_cursor_t cursor; - unw_context_t uc = *ucontext; - //if (recursive) { - // return 0; - //} - if (!custom_sanity_check()) { - return 0; - } - //++recursive; - - int ret = unw_init_local(&cursor, &uc); - assert(ret >= 0); - _unused(ret); - int first_run = 1; - - while (n < max_depth) { - if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { - break; - } - - unw_proc_info_t pip; - unw_get_proc_info(&cursor, &pip); - - /* char funcname[4096]; */ - /* unw_word_t offset; */ - /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */ - /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */ - - /* if n==0, it means that the signal handler interrupted us while we - were in the trampoline, so we are not executing (yet) the real main - loop function; just skip it */ - if (vmprof_mainloop_func && - (void*)pip.start_ip == (void*)vmprof_mainloop_func && - n > 0) { - // found main loop stack frame - void* sp; - unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp); - void *arg_addr = (char*)sp + mainloop_sp_offset; - void **arg_ptr = (void**)arg_addr; - // fprintf(stderr, "stacktrace mainloop: rsp %p &f2 %p offset %ld\n", - // sp, arg_addr, mainloop_sp_offset); - if (mainloop_get_virtual_ip) { - ip = mainloop_get_virtual_ip(*arg_ptr); - } else { - ip = *arg_ptr; - } - } - - result[n++] = ip; - n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth); - if (vmprof_unw_step(&cursor, first_run) <= 0) { - break; - } - first_run = 0; - } - //--recursive; - return n; -} - - -static int __attribute__((noinline)) frame_forcer(int rv) { - return rv; -} - -static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { - void* stack[MAX_STACK_DEPTH]; - int saved_errno = errno; - stack[0] = GetPC((ucontext_t*)ucontext); - int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); - depth++; // To account for pc value in stack[0]; - prof_write_stacktrace(stack, depth, 1); - errno = saved_errno; -} - -/* ************************************************************* - * functions to enable/disable the profiler - * ************************************************************* - */ - -static int open_profile(int fd, long period_usec, int write_header, char *s, - int slen) { - if ((fd = dup(fd)) == -1) { - return -1; - } - profile_buffer_position = 0; - profile_file = fd; - if (write_header) - prof_header(period_usec); - if (s) - write(profile_file, s, slen); - return 0; -} - -static int close_profile(void) { - // XXX all of this can happily fail - FILE* src; - char buf[BUFSIZ]; - size_t size; - int marker = MARKER_TRAILER; - write(profile_file, &marker, 1); - -#ifdef __linux__ - // copy /proc/PID/maps to the end of the profile file - sprintf(buf, "/proc/%d/maps", getpid()); - src = fopen(buf, "r"); - if (!src) { - vmprof_error = "error opening proc maps"; - return -1; - } - while ((size = fread(buf, 1, BUFSIZ, src))) { - write(profile_file, buf, size); - } - fclose(src); -#else - // freebsd and mac - sprintf(buf, "procstat -v %d", getpid()); - src = popen(buf, "r"); - if (!src) { - vmprof_error = "error calling procstat"; - return -1; - } - while ((size = fread(buf, 1, BUFSIZ, src))) { - write(profile_file, buf, size); - } - pclose(src); -#endif - close(profile_file); - return 0; -} - - -static int install_sigprof_handler(void) { - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_sigaction = sigprof_handler; - sa.sa_flags = SA_RESTART | SA_SIGINFO; - if (sigemptyset(&sa.sa_mask) == -1 || - sigaction(SIGPROF, &sa, NULL) == -1) { - return -1; - } - return 0; -} - -static int remove_sigprof_handler(void) { - sighandler_t res = signal(SIGPROF, SIG_DFL); - if (res == SIG_ERR) { - return -1; - } - return 0; -}; - -static int install_sigprof_timer(long period_usec) { - static struct itimerval timer; - last_period_usec = period_usec; - timer.it_interval.tv_sec = 0; - timer.it_interval.tv_usec = period_usec; - timer.it_value = timer.it_interval; - if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { - return -1; - } - return 0; -} - -static int remove_sigprof_timer(void) { - static struct itimerval timer; - timer.it_interval.tv_sec = 0; - timer.it_interval.tv_usec = 0; - timer.it_value.tv_sec = 0; - timer.it_value.tv_usec = 0; - if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { - return -1; - } - return 0; -} - -static void atfork_disable_timer(void) { - if (last_period_usec) { - remove_sigprof_timer(); - } -} - -static void atfork_enable_timer(void) { - if (last_period_usec) { - install_sigprof_timer(last_period_usec); - } -} - -static int install_pthread_atfork_hooks(void) { - /* this is needed to prevent the problems described there: - - http://code.google.com/p/gperftools/issues/detail?id=278 - - http://lists.debian.org/debian-glibc/2010/03/msg00161.html - - TL;DR: if the RSS of the process is large enough, the clone() syscall - will be interrupted by the SIGPROF before it can complete, then - retried, interrupted again and so on, in an endless loop. The - solution is to disable the timer around the fork, and re-enable it - only inside the parent. - */ - if (atfork_hook_installed) - return 0; - int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); - if (ret != 0) - return -1; - atfork_hook_installed = 1; - return 0; -} - -/* ************************************************************* - * public API - * ************************************************************* - */ - -int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, - vmprof_get_virtual_ip_t get_virtual_ip) { - void *libhandle; - - mainloop_sp_offset = sp_offset; - mainloop_get_virtual_ip = get_virtual_ip; - vmprof_mainloop_func = func; - if (!unw_get_reg) { - if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) { - vmprof_error = dlerror(); - return -1; - } - if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) { - vmprof_error = dlerror(); - return -1; - } - if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))){ - vmprof_error = dlerror(); - return -1; - } - if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) { - vmprof_error = dlerror(); - return -1; - } - if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) { - vmprof_error = dlerror(); - return -1; - } - } - return 0; -} - -char* vmprof_get_error() -{ - char* res; - res = vmprof_error; - vmprof_error = NULL; - return res; -} - -int vmprof_enable(int fd, long period_usec, int write_header, char *s, - int slen) -{ - assert(period_usec > 0); - if (open_profile(fd, period_usec, write_header, s, slen) == -1) { - return -1; - } - if (install_sigprof_handler() == -1) { - return -1; - } - if (install_sigprof_timer(period_usec) == -1) { - return -1; - } - if (install_pthread_atfork_hooks() == -1) { - return -1; - } - return 0; -} - -int vmprof_disable(void) { - if (remove_sigprof_timer() == -1) { - return -1; - } - last_period_usec = 0; - if (remove_sigprof_handler() == -1) { - return -1; - } - if (close_profile() == -1) { - return -1; - } - return 0; -} - -void vmprof_register_virtual_function(const char* name, void* start, void* end) { - // XXX unused by pypy - // for now *end is simply ignored - char buf[1024]; - int lgt = strlen(name) + 2 * sizeof(long) + 1; - - if (lgt > 1024) { - lgt = 1024; - } - buf[0] = MARKER_VIRTUAL_IP; - ((void **)(((void*)buf) + 1))[0] = start; - ((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1; - strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1); - write(profile_file, buf, lgt); -} diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h deleted file mode 100644 --- a/pypy/module/_vmprof/src/vmprof.h +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef VMPROF_VMPROF_H_ -#define VMPROF_VMPROF_H_ - -#include -#include -#include - -// copied from libunwind.h - -typedef enum - { - UNW_X86_64_RAX, - UNW_X86_64_RDX, - UNW_X86_64_RCX, - UNW_X86_64_RBX, - UNW_X86_64_RSI, - UNW_X86_64_RDI, - UNW_X86_64_RBP, - UNW_X86_64_RSP, - UNW_X86_64_R8, - UNW_X86_64_R9, - UNW_X86_64_R10, - UNW_X86_64_R11, - UNW_X86_64_R12, - UNW_X86_64_R13, - UNW_X86_64_R14, - UNW_X86_64_R15, - UNW_X86_64_RIP, -#ifdef CONFIG_MSABI_SUPPORT - UNW_X86_64_XMM0, - UNW_X86_64_XMM1, - UNW_X86_64_XMM2, - UNW_X86_64_XMM3, - UNW_X86_64_XMM4, - UNW_X86_64_XMM5, - UNW_X86_64_XMM6, - UNW_X86_64_XMM7, - UNW_X86_64_XMM8, - UNW_X86_64_XMM9, - UNW_X86_64_XMM10, - UNW_X86_64_XMM11, - UNW_X86_64_XMM12, - UNW_X86_64_XMM13, - UNW_X86_64_XMM14, - UNW_X86_64_XMM15, - UNW_TDEP_LAST_REG = UNW_X86_64_XMM15, -#else - UNW_TDEP_LAST_REG = UNW_X86_64_RIP, -#endif - - /* XXX Add other regs here */ - - /* frame info (read-only) */ - UNW_X86_64_CFA, - - UNW_TDEP_IP = UNW_X86_64_RIP, - UNW_TDEP_SP = UNW_X86_64_RSP, - UNW_TDEP_BP = UNW_X86_64_RBP, - UNW_TDEP_EH = UNW_X86_64_RAX - } -x86_64_regnum_t; - -typedef uint64_t unw_word_t; - -#define UNW_TDEP_CURSOR_LEN 127 - -typedef struct unw_cursor - { - unw_word_t opaque[UNW_TDEP_CURSOR_LEN]; - } -unw_cursor_t; - -#define UNW_REG_IP UNW_X86_64_RIP -#define UNW_REG_SP UNW_X86_64_RSP - -typedef ucontext_t unw_context_t; - -typedef struct unw_proc_info - { - unw_word_t start_ip; /* first IP covered by this procedure */ - unw_word_t end_ip; /* first IP NOT covered by this procedure */ - unw_word_t lsda; /* address of lang.-spec. data area (if any) */ - unw_word_t handler; /* optional personality routine */ - unw_word_t gp; /* global-pointer value for this procedure */ - unw_word_t flags; /* misc. flags */ - - int format; /* unwind-info format (arch-specific) */ - int unwind_info_size; /* size of the information (if applicable) */ - void *unwind_info; /* unwind-info (arch-specific) */ - } -unw_proc_info_t; - -// functions copied from libunwind using dlopen - -extern int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*); -extern int (*unw_step)(unw_cursor_t*); -extern int (*unw_init_local)(unw_cursor_t *, unw_context_t *); -extern int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *); - -// end of copy - -extern char* vmprof_error; - -typedef void* (*vmprof_get_virtual_ip_t)(void*); -char* vmprof_get_error(); - -extern void* vmprof_mainloop_func; -int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, - vmprof_get_virtual_ip_t get_virtual_ip); - -void vmprof_register_virtual_function(const char* name, void* start, void* end); - - -int vmprof_enable(int fd, long period_usec, int write_header, char* vips, - int vips_len); -int vmprof_disable(void); - -// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library) -void vmprof_set_tramp_range(void* start, void* end); - -#endif diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -1,14 +1,14 @@ -import tempfile +from rpython.tool.udir import udir from pypy.tool.pytest.objspace import gettestobjspace class AppTestVMProf(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) - cls.tmpfile = tempfile.NamedTemporaryFile() + cls.tmpfile = udir.join('test__vmprof.1').open('wb') cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno()) cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name) - cls.tmpfile2 = tempfile.NamedTemporaryFile() + cls.tmpfile2 = udir.join('test__vmprof.2').open('wb') cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno()) cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name) @@ -29,19 +29,23 @@ while i < len(s): if s[i] == '\x03': break - if s[i] == '\x01': - xxx - assert s[i] == '\x02' - i += 1 - _, size = struct.unpack("ll", s[i:i + 2 * WORD]) - count += 1 - i += 2 * WORD + size + elif s[i] == '\x01': + i += 1 + _, size = struct.unpack("ll", s[i:i + 2 * WORD]) + i += 2 * WORD + size * struct.calcsize("P") + elif s[i] == '\x02': + i += 1 + _, size = struct.unpack("ll", s[i:i + 2 * WORD]) + count += 1 + i += 2 * WORD + size + else: + raise AssertionError(ord(s[i])) return count import _vmprof - _vmprof.enable(self.tmpfileno) + _vmprof.enable(self.tmpfileno, 0.01) _vmprof.disable() - s = open(self.tmpfilename).read() + s = open(self.tmpfilename, 'rb').read() no_of_codes = count(s) assert no_of_codes > 10 d = {} @@ -50,14 +54,14 @@ pass """ in d - _vmprof.enable(self.tmpfileno2) + _vmprof.enable(self.tmpfileno2, 0.01) exec """def foo2(): pass """ in d _vmprof.disable() - s = open(self.tmpfilename2).read() + s = open(self.tmpfilename2, 'rb').read() no_of_codes2 = count(s) assert "py:foo:" in s assert "py:foo2:" in s @@ -65,8 +69,9 @@ def test_enable_ovf(self): import _vmprof - raises(ValueError, _vmprof.enable, 999, 0) - raises(ValueError, _vmprof.enable, 999, -2.5) - raises(ValueError, _vmprof.enable, 999, 1e300) - raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300) - raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300)) + raises(_vmprof.VMProfError, _vmprof.enable, 999, 0) + raises(_vmprof.VMProfError, _vmprof.enable, 999, -2.5) + raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300) + raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300 * 1e300) + NaN = (1e300*1e300) / (1e300*1e300) + raises(_vmprof.VMProfError, _vmprof.enable, 999, NaN) diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py --- a/pypy/module/_vmprof/test/test_direct.py +++ b/pypy/module/_vmprof/test/test_direct.py @@ -5,7 +5,8 @@ except ImportError: py.test.skip('cffi required') -srcdir = py.path.local(__file__).join("..", "..", "src") +from rpython.rlib import rvmprof +srcdir = py.path.local(rvmprof.__file__).join("..", "src") ffi = cffi.FFI() ffi.cdef(""" @@ -17,6 +18,8 @@ """) lib = ffi.verify(""" +#define PYPY_JIT_CODEMAP + volatile int pypy_codemap_currently_invalid = 0; long buffer[] = {0, 0, 0, 0, 0}; @@ -39,7 +42,7 @@ } -""" + open(str(srcdir.join("get_custom_offset.c"))).read()) +""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read()) class TestDirect(object): def test_infrastructure(self): diff --git a/pypy/module/cpyext/TODO b/pypy/module/cpyext/TODO deleted file mode 100644 --- a/pypy/module/cpyext/TODO +++ /dev/null @@ -1,26 +0,0 @@ - - Complete the PyTypeObject initialization code. (see XXX in the code) - - Implement further method callers. - - Copy the slots from the base. - - Those tasks are necessary to be able to call slots from C code correctly. - - Additionally, implement further slot wrappers. This is necessary to call - slots of PTOs defined in C. - - Complete the Members support. - - - Use a WeakKeyDictionary to count how often a PyObject is allocated for - a given wrapped object and use this to assess whether optimizations are - useful - - - replace @cpython_api(external=False) by another explicit name: all - it does is a lltype function pointer, no C code involved. - - - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). - - - refactor management of py_objects_r2w and py_objects_w2r, this can - probably be expressed in terms of _PyObject_GC_TRACK macros. - - - PyWeakref_GetObject() returns a borrowed reference, but this turns the - WeakRef into a strong reference! - - - sort out pypy's buffer protocol. PyPy's buffer right now don't support - raw memory (except array which supports it in a hackish way), which - should be fixed in order to make it nicely work with cpyext. diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py --- a/pypy/module/micronumpy/casting.py +++ b/pypy/module/micronumpy/casting.py @@ -145,23 +145,32 @@ # equivalent to PyArray_CanCastTypeTo if origin == target: return True - if origin.is_record() or target.is_record(): - return can_cast_record(space, origin, target, casting) + if casting == 'unsafe': + return True + elif casting == 'no': + return origin.eq(space, target) + if origin.num == target.num: + if origin.is_record(): + return (target.is_record() and + can_cast_record(space, origin, target, casting)) + else: + if casting == 'equiv': + return origin.elsize == target.elsize + elif casting == 'safe': + return origin.elsize <= target.elsize + else: + return True - if casting == 'no': - return origin.eq(space, target) - elif casting == 'equiv': - return origin.num == target.num and origin.elsize == target.elsize - elif casting == 'unsafe': - return True elif casting == 'same_kind': if can_cast_to(origin, target): return True if origin.kind in kind_ordering and target.kind in kind_ordering: return kind_ordering[origin.kind] <= kind_ordering[target.kind] return False - else: # 'safe' + elif casting == 'safe': return can_cast_to(origin, target) + else: # 'equiv' + return origin.num == target.num and origin.elsize == target.elsize def can_cast_record(space, origin, target, casting): if origin is target: diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -101,6 +101,9 @@ @specialize.argtype(1) def box(self, value): + if self.is_record(): + raise oefmt(self.itemtype.space.w_NotImplementedError, + "cannot box a value into a 'record' dtype, this is a bug please report it") return self.itemtype.box(value) @specialize.argtype(1, 2) @@ -1028,6 +1031,11 @@ elif space.isinstance_w(w_dtype, space.w_tuple): w_dtype0 = space.getitem(w_dtype, space.wrap(0)) w_dtype1 = space.getitem(w_dtype, space.wrap(1)) + if space.isinstance_w(w_dtype0, space.w_type) and \ + space.isinstance_w(w_dtype1, space.w_list): + #obscure api - (subclass, spec). Ignore the subclass + return make_new_dtype(space, w_subtype, w_dtype1, alignment, + copy=copy, w_shape=w_shape, w_metadata=w_metadata) subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy) assert isinstance(subdtype, W_Dtype) if subdtype.elsize == 0: diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py --- a/pypy/module/micronumpy/iterators.py +++ b/pypy/module/micronumpy/iterators.py @@ -204,17 +204,16 @@ self.array.setitem(state.offset, elem) -def AxisIter(array, shape, axis, cumulative): +def AxisIter(array, shape, axis): strides = array.get_strides() backstrides = array.get_backstrides() - if not cumulative: - if len(shape) == len(strides): - # keepdims = True - strides = strides[:axis] + [0] + strides[axis + 1:] - backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:] - else: - strides = strides[:axis] + [0] + strides[axis:] - backstrides = backstrides[:axis] + [0] + backstrides[axis:] + if len(shape) == len(strides): + # keepdims = True + strides = strides[:axis] + [0] + strides[axis + 1:] + backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:] + else: + strides = strides[:axis] + [0] + strides[axis:] + backstrides = backstrides[:axis] + [0] + backstrides[axis:] return ArrayIter(array, support.product(shape), shape, strides, backstrides) diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -9,7 +9,7 @@ from pypy.module.micronumpy import support, constants as NPY from pypy.module.micronumpy.base import W_NDimArray, convert_to_array from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \ - AllButAxisIter + AllButAxisIter, ArrayIter from pypy.interpreter.argument import Arguments @@ -190,23 +190,64 @@ source_state = source_iter.next(source_state) return target -reduce_driver = jit.JitDriver(name='numpy_reduce', - greens = ['shapelen', 'func', 'done_func', - 'calc_dtype'], - reds = 'auto') -def compute_reduce(space, obj, calc_dtype, func, done_func, identity): - obj_iter, obj_state = obj.create_iter() +def split_iter(arr, axis_flags): + """Prepare 2 iterators for nested iteration over `arr`. + + Arguments: + arr: instance of BaseConcreteArray + axis_flags: list of bools, one for each dimension of `arr`.The inner + iterator operates over the dimensions for which the flag is True + """ + shape = arr.get_shape() + strides = arr.get_strides() + backstrides = arr.get_backstrides() + shapelen = len(shape) + assert len(axis_flags) == shapelen + inner_shape = [-1] * shapelen + inner_strides = [-1] * shapelen + inner_backstrides = [-1] * shapelen + outer_shape = [-1] * shapelen + outer_strides = [-1] * shapelen + outer_backstrides = [-1] * shapelen + for i in range(len(shape)): + if axis_flags[i]: + inner_shape[i] = shape[i] + inner_strides[i] = strides[i] + inner_backstrides[i] = backstrides[i] + outer_shape[i] = 1 + outer_strides[i] = 0 + outer_backstrides[i] = 0 + else: + outer_shape[i] = shape[i] + outer_strides[i] = strides[i] + outer_backstrides[i] = backstrides[i] + inner_shape[i] = 1 + inner_strides[i] = 0 + inner_backstrides[i] = 0 + inner_iter = ArrayIter(arr, support.product(inner_shape), + inner_shape, inner_strides, inner_backstrides) + outer_iter = ArrayIter(arr, support.product(outer_shape), + outer_shape, outer_strides, outer_backstrides) + return inner_iter, outer_iter + + +reduce_flat_driver = jit.JitDriver( + name='numpy_reduce_flat', + greens = ['shapelen', 'func', 'done_func', 'calc_dtype'], reds = 'auto') + +def reduce_flat(space, func, w_arr, calc_dtype, done_func, identity): + obj_iter, obj_state = w_arr.create_iter() if identity is None: cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) obj_state = obj_iter.next(obj_state) else: cur_value = identity.convert_to(space, calc_dtype) - shapelen = len(obj.get_shape()) + shapelen = len(w_arr.get_shape()) while not obj_iter.done(obj_state): - reduce_driver.jit_merge_point(shapelen=shapelen, func=func, - done_func=done_func, - calc_dtype=calc_dtype) + reduce_flat_driver.jit_merge_point( + shapelen=shapelen, func=func, + done_func=done_func, calc_dtype=calc_dtype) rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) if done_func is not None and done_func(calc_dtype, rval): return rval @@ -214,33 +255,105 @@ obj_state = obj_iter.next(obj_state) return cur_value -reduce_cum_driver = jit.JitDriver( - name='numpy_reduce_cum_driver', + +reduce_driver = jit.JitDriver( + name='numpy_reduce', + greens=['shapelen', 'func', 'dtype'], reds='auto') + +def reduce(space, func, w_arr, axis_flags, dtype, out, identity): + out_iter, out_state = out.create_iter() + out_iter.track_index = False + shape = w_arr.get_shape() + shapelen = len(shape) + inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags) + assert outer_iter.size == out_iter.size + + if identity is not None: + identity = identity.convert_to(space, dtype) + outer_state = outer_iter.reset() + while not outer_iter.done(outer_state): + inner_state = inner_iter.reset() + inner_state.offset = outer_state.offset + if identity is not None: + w_val = identity + else: + w_val = inner_iter.getitem(inner_state).convert_to(space, dtype) + inner_state = inner_iter.next(inner_state) + while not inner_iter.done(inner_state): + reduce_driver.jit_merge_point( + shapelen=shapelen, func=func, dtype=dtype) + w_item = inner_iter.getitem(inner_state).convert_to(space, dtype) + w_val = func(dtype, w_item, w_val) + inner_state = inner_iter.next(inner_state) + out_iter.setitem(out_state, w_val) + out_state = out_iter.next(out_state) + outer_state = outer_iter.next(outer_state) + return out + +accumulate_flat_driver = jit.JitDriver( + name='numpy_accumulate_flat', greens=['shapelen', 'func', 'dtype', 'out_dtype'], reds='auto') -def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity): - obj_iter, obj_state = obj.create_iter() - out_iter, out_state = out.create_iter() +def accumulate_flat(space, func, w_arr, calc_dtype, w_out, identity): + arr_iter, arr_state = w_arr.create_iter() + out_iter, out_state = w_out.create_iter() out_iter.track_index = False if identity is None: - cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) + cur_value = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) out_iter.setitem(out_state, cur_value) out_state = out_iter.next(out_state) - obj_state = obj_iter.next(obj_state) + arr_state = arr_iter.next(arr_state) else: cur_value = identity.convert_to(space, calc_dtype) - shapelen = len(obj.get_shape()) - out_dtype = out.get_dtype() - while not obj_iter.done(obj_state): - reduce_cum_driver.jit_merge_point( - shapelen=shapelen, func=func, - dtype=calc_dtype, out_dtype=out_dtype) - rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) - cur_value = func(calc_dtype, cur_value, rval) + shapelen = len(w_arr.get_shape()) + out_dtype = w_out.get_dtype() + while not arr_iter.done(arr_state): + accumulate_flat_driver.jit_merge_point( + shapelen=shapelen, func=func, dtype=calc_dtype, + out_dtype=out_dtype) + w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) + cur_value = func(calc_dtype, cur_value, w_item) out_iter.setitem(out_state, out_dtype.coerce(space, cur_value)) out_state = out_iter.next(out_state) - obj_state = obj_iter.next(obj_state) + arr_state = arr_iter.next(arr_state) + +accumulate_driver = jit.JitDriver( + name='numpy_accumulate', + greens=['shapelen', 'func', 'calc_dtype'], reds='auto') + + +def accumulate(space, func, w_arr, axis, calc_dtype, w_out, identity): + out_iter, out_state = w_out.create_iter() + arr_shape = w_arr.get_shape() + temp_shape = arr_shape[:axis] + arr_shape[axis + 1:] + temp = W_NDimArray.from_shape(space, temp_shape, calc_dtype, w_instance=w_arr) + temp_iter = AxisIter(temp.implementation, w_arr.get_shape(), axis) + temp_state = temp_iter.reset() + arr_iter, arr_state = w_arr.create_iter() + arr_iter.track_index = False + if identity is not None: + identity = identity.convert_to(space, calc_dtype) + shapelen = len(arr_shape) + while not out_iter.done(out_state): + accumulate_driver.jit_merge_point(shapelen=shapelen, func=func, + calc_dtype=calc_dtype) + w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) + arr_state = arr_iter.next(arr_state) + + out_indices = out_iter.indices(out_state) + if out_indices[axis] == 0: + if identity is not None: + w_item = func(calc_dtype, identity, w_item) + else: + cur_value = temp_iter.getitem(temp_state) + w_item = func(calc_dtype, cur_value, w_item) + + out_iter.setitem(out_state, w_item) + out_state = out_iter.next(out_state) + temp_iter.setitem(temp_state, w_item) + temp_state = temp_iter.next(temp_state) + return w_out def fill(arr, box): arr_iter, arr_state = arr.create_iter() @@ -298,64 +411,56 @@ state = x_state return out -axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', 'func', 'dtype'], - reds='auto') - -def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative, - temp): - out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative) - out_state = out_iter.reset() - if cumulative: - temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False) - temp_state = temp_iter.reset() - else: - temp_iter = out_iter # hack - temp_state = out_state - arr_iter, arr_state = arr.create_iter() - arr_iter.track_index = False - if identity is not None: - identity = identity.convert_to(space, dtype) - shapelen = len(shape) - while not out_iter.done(out_state): - axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func, - dtype=dtype) - w_val = arr_iter.getitem(arr_state).convert_to(space, dtype) - arr_state = arr_iter.next(arr_state) - - out_indices = out_iter.indices(out_state) - if out_indices[axis] == 0: - if identity is not None: - w_val = func(dtype, identity, w_val) - else: - cur = temp_iter.getitem(temp_state) - w_val = func(dtype, cur, w_val) - - out_iter.setitem(out_state, w_val) - out_state = out_iter.next(out_state) - if cumulative: - temp_iter.setitem(temp_state, w_val) - temp_state = temp_iter.next(temp_state) - else: - temp_state = out_state - return out - def _new_argmin_argmax(op_name): arg_driver = jit.JitDriver(name='numpy_' + op_name, greens = ['shapelen', 'dtype'], reds = 'auto') + arg_flat_driver = jit.JitDriver(name='numpy_flat_' + op_name, + greens = ['shapelen', 'dtype'], + reds = 'auto') - def argmin_argmax(arr): + def argmin_argmax(space, w_arr, w_out, axis): + from pypy.module.micronumpy.descriptor import get_dtype_cache + dtype = w_arr.get_dtype() + shapelen = len(w_arr.get_shape()) + axis_flags = [False] * shapelen + axis_flags[axis] = True + inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags) + outer_state = outer_iter.reset() + out_iter, out_state = w_out.create_iter() + while not outer_iter.done(outer_state): + inner_state = inner_iter.reset() + inner_state.offset = outer_state.offset + cur_best = inner_iter.getitem(inner_state) + inner_state = inner_iter.next(inner_state) + result = 0 + idx = 1 + while not inner_iter.done(inner_state): + arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) + w_val = inner_iter.getitem(inner_state) + new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) + if dtype.itemtype.ne(new_best, cur_best): + result = idx + cur_best = new_best + inner_state = inner_iter.next(inner_state) + idx += 1 + result = get_dtype_cache(space).w_longdtype.box(result) + out_iter.setitem(out_state, result) + out_state = out_iter.next(out_state) + outer_state = outer_iter.next(outer_state) + return w_out + + def argmin_argmax_flat(w_arr): result = 0 idx = 1 - dtype = arr.get_dtype() - iter, state = arr.create_iter() + dtype = w_arr.get_dtype() + iter, state = w_arr.create_iter() cur_best = iter.getitem(state) state = iter.next(state) - shapelen = len(arr.get_shape()) + shapelen = len(w_arr.get_shape()) while not iter.done(state): - arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) + arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) w_val = iter.getitem(state) new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) if dtype.itemtype.ne(new_best, cur_best): @@ -364,9 +469,10 @@ state = iter.next(state) idx += 1 return result - return argmin_argmax -argmin = _new_argmin_argmax('min') -argmax = _new_argmin_argmax('max') + + return argmin_argmax, argmin_argmax_flat +argmin, argmin_flat = _new_argmin_argmax('min') +argmax, argmax_flat = _new_argmin_argmax('max') dot_driver = jit.JitDriver(name = 'numpy_dot', greens = ['dtype'], diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -23,6 +23,8 @@ get_shape_from_iterable, shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous, calc_strides, new_view) from pypy.module.micronumpy.casting import can_cast_array +from pypy.module.micronumpy.descriptor import get_dtype_cache + def _match_dot_shapes(space, left, right): @@ -484,7 +486,7 @@ return self.implementation.swapaxes(space, self, axis1, axis2) def descr_nonzero(self, space): - index_type = descriptor.get_dtype_cache(space).w_int64dtype + index_type = get_dtype_cache(space).w_int64dtype return self.implementation.nonzero(space, index_type) def descr_tolist(self, space): @@ -544,8 +546,10 @@ def descr_set_flatiter(self, space, w_obj): iter, state = self.create_iter() dtype = self.get_dtype() - arr = convert_to_array(space, w_obj) - loop.flatiter_setitem(space, dtype, arr, iter, state, 1, iter.size) + w_arr = convert_to_array(space, w_obj) + if dtype.is_record(): + return self.implementation.setslice(space, w_arr) + loop.flatiter_setitem(space, dtype, w_arr, iter, state, 1, iter.size) def descr_get_flatiter(self, space): from .flatiter import W_FlatIterator @@ -810,7 +814,7 @@ if self.get_dtype().is_bool(): # numpy promotes bool.round() to float16. Go figure. w_out = W_NDimArray.from_shape(space, self.get_shape(), - descriptor.get_dtype_cache(space).w_float16dtype) + get_dtype_cache(space).w_float16dtype) else: w_out = None elif not isinstance(w_out, W_NDimArray): @@ -818,7 +822,7 @@ "return arrays must be of ArrayType")) out = descriptor.dtype_agreement(space, [self], self.get_shape(), w_out) if out.get_dtype().is_bool() and self.get_dtype().is_bool(): - calc_dtype = descriptor.get_dtype_cache(space).w_longdtype + calc_dtype = get_dtype_cache(space).w_longdtype else: calc_dtype = out.get_dtype() @@ -837,7 +841,7 @@ raise oefmt(space.w_ValueError, "a must be a 1-d array") v = convert_to_array(space, w_v) ret = W_NDimArray.from_shape( - space, v.get_shape(), descriptor.get_dtype_cache(space).w_longdtype) + space, v.get_shape(), get_dtype_cache(space).w_longdtype) if side == NPY.SEARCHLEFT: binsearch = loop.binsearch_left else: @@ -1145,35 +1149,46 @@ # ----------------------- reduce ------------------------------- - def _reduce_ufunc_impl(ufunc_name, cumulative=False, bool_result=False): + def _reduce_ufunc_impl(ufunc_name, name, bool_result=False): @unwrap_spec(keepdims=bool) def impl(self, space, w_axis=None, w_dtype=None, w_out=None, keepdims=False): out = out_converter(space, w_out) if bool_result: - w_dtype = descriptor.get_dtype_cache(space).w_booldtype + w_dtype = get_dtype_cache(space).w_booldtype return getattr(ufuncs.get(space), ufunc_name).reduce( From noreply at buildbot.pypy.org Tue Aug 11 15:27:02 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 15:27:02 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): use a slightly different approach, both a bit more flexible Message-ID: <20150811132702.8909E1C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78894:9c96634eb883 Date: 2015-08-11 14:44 +0200 http://bitbucket.org/pypy/pypy/changeset/9c96634eb883/ Log: (arigo, cfbolz): use a slightly different approach, both a bit more flexible and safer diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -50,14 +50,23 @@ kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None return Signature(argnames, varargname, kwargname) +class ValueProf(valueprof.ValueProf): + def is_int(self, w_obj): + from pypy.objspace.std.intobject import W_IntObject + return type(w_obj) is W_IntObject + + def get_int_val(self, w_obj): + from pypy.objspace.std.intobject import W_IntObject + assert isinstance(w_obj, W_IntObject) + return w_obj.intval class PyCode(eval.Code): "CPython-style code objects." _immutable_ = True _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]", "co_freevars[*]", "co_cellvars[*]", - "_args_as_cellvars[*]", "vprof"] - + "_args_as_cellvars[*]", "vprofs[*]"] + def __init__(self, space, argcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars, @@ -86,8 +95,7 @@ self._signature = cpython_code_signature(self) self._initialize() space.register_code_object(self) - self.vprof = valueprof.ValueProf(self.co_nlocals) - self.printed = [False] * self.co_nlocals + self.vprofs = [ValueProf() for i in range(self.co_nlocals)] def _initialize(self): if self.co_cellvars: diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -147,22 +147,16 @@ def _getlocal(self, varindex): from pypy.objspace.std.intobject import W_IntObject + if we_are_jitted(): + vprof = self.getcode().vprofs[varindex] + # some careful logic there + if vprof.can_fold_read_int(): + return W_IntObject(vprof.read_constant_int()) + elif vprof.can_fold_read_obj(): + w_res = vprof.try_read_constant_obj() + if w_res is not None: + return w_res w_res = self.locals_cells_stack_w[varindex] - if we_are_jitted(): - vprof = self.getcode().vprof - # some careful logic there - frozen = vprof.freeze() - assert frozen - if vprof.is_variable_constant(varindex): - if vprof.is_variable_int(varindex): - res_prof = vprof.variable_value_int(varindex) - if isinstance(w_res, W_IntObject): - if w_res.intval == res_prof: - return W_IntObject(res_prof) - else: - w_res_prof = vprof.variable_value_object(varindex) - if w_res is w_res_prof: - return w_res_prof return w_res def _setlocal(self, varindex, value): @@ -171,16 +165,8 @@ def _value_profile_local(self, varindex, value): from pypy.objspace.std.intobject import W_IntObject - if we_are_jitted(): - return - vprof = self.pycode.vprof - if vprof.frozen: - return - if isinstance(value, W_IntObject): - times = vprof.see_int(varindex, value.intval) - else: - times = self.pycode.vprof.see_object(varindex, value) - + vprof = self.getcode().vprofs[varindex] + vprof.see_write(value) def mark_as_escaped(self): """ diff --git a/pypy/interpreter/test/test_valueprof.py b/pypy/interpreter/test/test_valueprof.py --- a/pypy/interpreter/test/test_valueprof.py +++ b/pypy/interpreter/test/test_valueprof.py @@ -1,45 +1,85 @@ -from pypy.interpreter.valueprof import ValueProf +from pypy.interpreter.valueprof import * class Value(): pass -def test_simple(): - v = ValueProf(2) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - assert v.values_int[0] == 1 - assert v.counters[0] == -4 - v.see_int(0, 5) - v.see_int(0, 5) - v.see_int(0, 5) - v.see_int(0, 5) - v.see_int(0, 5) - assert v.values_int[0] == 5 - assert v.counters[0] == -5 +class ValueInt(Value): + def __init__(self, val): + self.intval = val - val1 = Value() - v.see_object(0, val1) - v.see_object(0, val1) - v.see_object(0, val1) - v.see_object(0, val1) - assert v.values_wref[0]() is val1 - assert v.counters[0] == 4 - v.see_object(0, None) - assert v.counters[0] == 0 +class ValueProf(ValueProf): + def is_int(self, val): + return isinstance(val, ValueInt) -def test_freeze(): - v = ValueProf(2) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - v.see_int(0, 1) - v.freeze() - v.see_int(0, 2) - assert v.values_int[0] == 1 + def get_int_val(self, val): + return val.intval + + +def test_int(): + v = ValueProf() + assert v.status == SEEN_NOTHING + v.see_write(ValueInt(1)) + v.see_write(ValueInt(1)) + v.see_write(ValueInt(1)) + v.see_write(ValueInt(1)) + assert v.read_constant_int() == 1 + assert v.status == SEEN_INT + v.see_int(2) + assert v.status == SEEN_TOO_MUCH + v.see_int(1) + assert v.status == SEEN_TOO_MUCH + v.see_int(2) + assert v.status == SEEN_TOO_MUCH + v.see_int(3) + assert v.status == SEEN_TOO_MUCH + + v = ValueProf() + assert v.status == SEEN_NOTHING + v.see_write(ValueInt(1)) + v.see_write(Value()) + assert v.status == SEEN_TOO_MUCH + v.see_write(Value()) + assert v.status == SEEN_TOO_MUCH + + +def test_obj(): + v = ValueProf() + value = Value() + assert v.status == SEEN_NOTHING + v.see_write(value) + v.see_write(value) + v.see_write(value) + v.see_write(value) + assert v.try_read_constant_obj() is value + assert v.status == SEEN_OBJ + v.see_int(2) + assert v.status == SEEN_TOO_MUCH + + v = ValueProf() + assert v.status == SEEN_NOTHING + v.see_write(Value()) + v.see_write(Value()) + assert v.status == SEEN_TOO_MUCH + + +def test_none(): + v = ValueProf() + assert v.status == SEEN_NOTHING + v.see_write(None) + assert v.status == SEEN_TOO_MUCH + v.see_write(None) + assert v.status == SEEN_TOO_MUCH + + v = ValueProf() + v.see_write(ValueInt(1)) + assert v.status == SEEN_INT + v.see_write(None) + assert v.status == SEEN_TOO_MUCH + + v = ValueProf() + v.see_write(Value()) + assert v.status == SEEN_OBJ + v.see_write(None) + assert v.status == SEEN_TOO_MUCH diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -1,75 +1,74 @@ from rpython.rlib import jit from rpython.rlib.rweakref import ref, dead_ref +SEEN_NOTHING = '\x00' +SEEN_INT = '\x01' +SEEN_OBJ = '\x02' +SEEN_TOO_MUCH = '\x03' + class ValueProf(object): - def __init__(self, size, threshold=200): - self.values_wref = [dead_ref] * size - self.values_int = [-1] * size - self.counters = [0] * size - self.threshold = 200 - self.frozen = False + _mixin_ = True + _immutable_fields_ = ['status?'] + + def __init__(self): + # only if you subclass normally + self.init_valueprof() + + def init_valueprof(self): + self.status = SEEN_NOTHING + self.value_int = 0 + self.value_wref = dead_ref + + def is_int(self, w_obj): + raise NotImplementedError("abstract base") + + def get_int_val(self, w_obj): + raise NotImplementedError("abstract base") + + def see_write(self, w_value): + if self.is_int(w_value): + return self.see_int(self.get_int_val(w_value)) + return self.see_object(w_value) + + def see_int(self, value): + status = self.status + if status == SEEN_NOTHING: + self.value_int = value + self.status = SEEN_INT + elif status == SEEN_INT: + if self.read_constant_int() != value: + self.status = SEEN_TOO_MUCH + elif status == SEEN_OBJ: + self.status = SEEN_TOO_MUCH + + def see_object(self, value): + status = self.status + if value is None: + if status != SEEN_TOO_MUCH: + self.status = SEEN_TOO_MUCH + elif status == SEEN_NOTHING: + self.value_wref = ref(value) + self.status = SEEN_OBJ + elif status == SEEN_INT: + self.status = SEEN_TOO_MUCH + elif status == SEEN_OBJ: + if self.try_read_constant_obj() is not value: + self.status = SEEN_TOO_MUCH + + def can_fold_read_int(self): + return self.status == SEEN_INT + + def can_fold_read_obj(self): + return self.status == SEEN_OBJ @jit.elidable - def freeze(self): - # this works because we only ever change it in one direction - self.frozen = True - return True - - def see_int(self, index, value): - if self.frozen: - return 0 - count = self.counters[index] - if count < 0: - if self.values_int[index] == value: - new_count = count - 1 - self.counters[index] = new_count - return -new_count - else: - self.values_wref[index] = dead_ref - self.values_int[index] = value - self.counters[index] = -1 - return 1 - - def see_object(self, index, value): - if self.frozen: - return 0 - if value is None: - self.values_wref[index] = dead_ref - self.counters[index] = 0 - return 0 - count = self.counters[index] - if count > 0: - if self.values_wref[index]() is value: - new_count = count + 1 - self.counters[index] = new_count - return new_count - else: - self.values_int[index] = -1 - self.values_wref[index] = ref(value) - self.counters[index] = 1 - return 1 + def read_constant_int(self): + assert self.can_fold_read_int() + return self.value_int @jit.elidable - def is_variable_constant(self, index): - assert self.frozen - counter = self.counters[index] - if counter > 0: - return counter > self.threshold - else: - return -counter > self.threshold + def try_read_constant_obj(self): + assert self.can_fold_read_obj() + return self.value_wref() - @jit.elidable - def is_variable_int(self, index): - assert self.frozen - assert self.is_variable_constant(index) - return self.counters[index] < 0 - @jit.elidable - def variable_value_int(self, index): - assert self.is_variable_int(index) - return self.values_int[index] - - @jit.elidable - def variable_value_object(self, index): - assert self.is_variable_constant(index) and not self.is_variable_int(index) - return self.values_wref[index]() From noreply at buildbot.pypy.org Tue Aug 11 15:27:04 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 15:27:04 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (cfbolz, arigo): rename attributes to lessen risk of conflict Message-ID: <20150811132704.B09E21C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78895:4c89f76ea898 Date: 2015-08-11 15:06 +0200 http://bitbucket.org/pypy/pypy/changeset/4c89f76ea898/ Log: (cfbolz, arigo): rename attributes to lessen risk of conflict diff --git a/pypy/interpreter/test/test_valueprof.py b/pypy/interpreter/test/test_valueprof.py --- a/pypy/interpreter/test/test_valueprof.py +++ b/pypy/interpreter/test/test_valueprof.py @@ -19,67 +19,67 @@ def test_int(): v = ValueProf() - assert v.status == SEEN_NOTHING + assert v._vprof_status == SEEN_NOTHING v.see_write(ValueInt(1)) v.see_write(ValueInt(1)) v.see_write(ValueInt(1)) v.see_write(ValueInt(1)) assert v.read_constant_int() == 1 - assert v.status == SEEN_INT + assert v._vprof_status == SEEN_INT v.see_int(2) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v.see_int(1) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v.see_int(2) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v.see_int(3) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v = ValueProf() - assert v.status == SEEN_NOTHING + assert v._vprof_status == SEEN_NOTHING v.see_write(ValueInt(1)) v.see_write(Value()) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v.see_write(Value()) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH def test_obj(): v = ValueProf() value = Value() - assert v.status == SEEN_NOTHING + assert v._vprof_status == SEEN_NOTHING v.see_write(value) v.see_write(value) v.see_write(value) v.see_write(value) assert v.try_read_constant_obj() is value - assert v.status == SEEN_OBJ + assert v._vprof_status == SEEN_OBJ v.see_int(2) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v = ValueProf() - assert v.status == SEEN_NOTHING + assert v._vprof_status == SEEN_NOTHING v.see_write(Value()) v.see_write(Value()) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH def test_none(): v = ValueProf() - assert v.status == SEEN_NOTHING + assert v._vprof_status == SEEN_NOTHING v.see_write(None) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v.see_write(None) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v = ValueProf() v.see_write(ValueInt(1)) - assert v.status == SEEN_INT + assert v._vprof_status == SEEN_INT v.see_write(None) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH v = ValueProf() v.see_write(Value()) - assert v.status == SEEN_OBJ + assert v._vprof_status == SEEN_OBJ v.see_write(None) - assert v.status == SEEN_TOO_MUCH + assert v._vprof_status == SEEN_TOO_MUCH diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -8,16 +8,16 @@ class ValueProf(object): _mixin_ = True - _immutable_fields_ = ['status?'] + _immutable_fields_ = ['_vprof_status?'] def __init__(self): # only if you subclass normally self.init_valueprof() def init_valueprof(self): - self.status = SEEN_NOTHING - self.value_int = 0 - self.value_wref = dead_ref + self._vprof_status = SEEN_NOTHING + self._vprof_value_int = 0 + self._vprof_value_wref = dead_ref def is_int(self, w_obj): raise NotImplementedError("abstract base") @@ -31,44 +31,44 @@ return self.see_object(w_value) def see_int(self, value): - status = self.status + status = self._vprof_status if status == SEEN_NOTHING: - self.value_int = value - self.status = SEEN_INT + self._vprof_value_int = value + self._vprof_status = SEEN_INT elif status == SEEN_INT: if self.read_constant_int() != value: - self.status = SEEN_TOO_MUCH + self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_OBJ: - self.status = SEEN_TOO_MUCH + self._vprof_status = SEEN_TOO_MUCH def see_object(self, value): - status = self.status + status = self._vprof_status if value is None: if status != SEEN_TOO_MUCH: - self.status = SEEN_TOO_MUCH + self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_NOTHING: - self.value_wref = ref(value) - self.status = SEEN_OBJ + self._vprof_value_wref = ref(value) + self._vprof_status = SEEN_OBJ elif status == SEEN_INT: - self.status = SEEN_TOO_MUCH + self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_OBJ: if self.try_read_constant_obj() is not value: - self.status = SEEN_TOO_MUCH + self._vprof_status = SEEN_TOO_MUCH def can_fold_read_int(self): - return self.status == SEEN_INT + return self._vprof_status == SEEN_INT def can_fold_read_obj(self): - return self.status == SEEN_OBJ + return self._vprof_status == SEEN_OBJ @jit.elidable def read_constant_int(self): assert self.can_fold_read_int() - return self.value_int + return self._vprof_value_int @jit.elidable def try_read_constant_obj(self): assert self.can_fold_read_obj() - return self.value_wref() + return self._vprof_value_wref() From noreply at buildbot.pypy.org Tue Aug 11 15:27:07 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 15:27:07 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: merge Message-ID: <20150811132707.278121C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78896:c6fd1f04a9e0 Date: 2015-08-11 15:27 +0200 http://bitbucket.org/pypy/pypy/changeset/c6fd1f04a9e0/ Log: merge diff too long, truncating to 2000 out of 5220 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -55,3 +55,11 @@ .. branch: nditer-revisited Implement nditer 'buffered' flag and fix some edge cases + +.. branch: ufunc-reduce + +Allow multiple axes in ufunc.reduce() + +.. branch: fix-tinylang-goals + +Update tinylang goals to match current rpython diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -11,7 +11,7 @@ INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag, - UserDelAction, CodeUniqueIds) + UserDelAction) from pypy.interpreter.error import OperationError, new_exception_class, oefmt from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary @@ -391,7 +391,6 @@ self.actionflag = ActionFlag() # changed by the signal module self.check_signal_action = None # changed by the signal module self.user_del_action = UserDelAction(self) - self.code_unique_ids = CodeUniqueIds() self._code_of_sys_exc_info = None # can be overridden to a subclass @@ -670,16 +669,6 @@ assert ec is not None return ec - def register_code_callback(self, callback): - cui = self.code_unique_ids - cui.code_callback = callback - - def register_code_object(self, pycode): - cui = self.code_unique_ids - if cui.code_callback is None: - return - cui.code_callback(self, pycode) - def _freeze_(self): return True diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -590,11 +590,3 @@ # there is no list of length n: if n is large, then the GC # will run several times while walking the list, but it will # see lower and lower memory usage, with no lower bound of n. - -class CodeUniqueIds(object): - def __init__(self): - if sys.maxint == 2147483647: - self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit - else: - self.code_unique_id = 0x7000000000000000 - self.code_callback = None diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -94,7 +94,7 @@ self.magic = magic self._signature = cpython_code_signature(self) self._initialize() - space.register_code_object(self) + self._init_ready() self.vprofs = [ValueProf() for i in range(self.co_nlocals)] def _initialize(self): @@ -137,14 +137,8 @@ from pypy.objspace.std.mapdict import init_mapdict_cache init_mapdict_cache(self) - cui = self.space.code_unique_ids - self._unique_id = cui.code_unique_id - cui.code_unique_id += 4 # so we have two bits that we can mark stuff - # with - - def _get_full_name(self): - return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno, - self.co_filename) + def _init_ready(self): + "This is a hook for the vmprof module, which overrides this method." def _cleanup_(self): if (self.magic == cpython_magic and diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -62,6 +62,7 @@ } interpleveldefs = { + 'attach_gdb' : 'interp_magic.attach_gdb', 'internal_repr' : 'interp_magic.internal_repr', 'bytebuffer' : 'bytebuffer.bytebuffer', 'identity_dict' : 'interp_identitydict.W_IdentityDict', @@ -100,8 +101,6 @@ def setup_after_space_initialization(self): """NOT_RPYTHON""" - if not self.space.config.translating: - self.extra_interpdef('interp_pdb', 'interp_magic.interp_pdb') if self.space.config.objspace.std.withmethodcachecounter: self.extra_interpdef('method_cache_counter', 'interp_magic.method_cache_counter') diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -15,12 +15,10 @@ return space.wrap('%r' % (w_object,)) -def interp_pdb(space): - """Run an interp-level pdb. - This is not available in translated versions of PyPy.""" - assert not we_are_translated() - import pdb - pdb.set_trace() +def attach_gdb(space): + """Run an interp-level gdb (or pdb when untranslated)""" + from rpython.rlib.debug import attach_gdb + attach_gdb() @unwrap_spec(name=str) diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -209,7 +209,7 @@ # EAGAIN after already some data was received, return it. # Note that we can get EAGAIN while there is buffered data # waiting; read that too. - if is_wouldblock_error(e): + if is_wouldblock_error(e.errno): m = stream.count_buffered_bytes() if m > 0: result.append(stream.read(min(n, m))) @@ -321,6 +321,10 @@ self.getstream() # check if the file is still open return os.isatty(self.fd) + def direct_readinto(self, w_rwbuffer): + from pypy.module._file.readinto import direct_readinto + return direct_readinto(self, w_rwbuffer) + # ____________________________________________________________ # # The 'file_' methods are the one exposed to app-level. @@ -413,6 +417,9 @@ Notice that when in non-blocking mode, less data than what was requested may be returned, even if no size parameter was given.""") + _decl(locals(), "readinto", + """readinto(buf) -> length. Read into the given read-write buffer.""") + _decl(locals(), "readline", """readline([size]) -> next line from the file, as a string. @@ -508,16 +515,6 @@ for w_line in lines: self.file_write(w_line) - def file_readinto(self, w_rwbuffer): - """readinto() -> Undocumented. Don't use this; it may go away.""" - # XXX not the most efficient solution as it doesn't avoid the copying - space = self.space - rwbuffer = space.writebuf_w(w_rwbuffer) - w_data = self.file_read(rwbuffer.getlength()) - data = space.str_w(w_data) - rwbuffer.setslice(0, data) - return space.wrap(len(data)) - # ____________________________________________________________ @@ -603,7 +600,6 @@ cls=W_File, doc="Support for 'print'."), __repr__ = interp2app(W_File.file__repr__), - readinto = interp2app(W_File.file_readinto), writelines = interp2app(W_File.file_writelines), __exit__ = interp2app(W_File.file__exit__), __weakref__ = make_weakref_descr(W_File), @@ -632,10 +628,10 @@ MAYBE_EAGAIN = getattr(errno, 'EAGAIN', None) MAYBE_EWOULDBLOCK = getattr(errno, 'EWOULDBLOCK', None) -def is_wouldblock_error(e): - if MAYBE_EAGAIN is not None and e.errno == MAYBE_EAGAIN: +def is_wouldblock_error(errno): + if MAYBE_EAGAIN is not None and errno == MAYBE_EAGAIN: return True - if MAYBE_EWOULDBLOCK is not None and e.errno == MAYBE_EWOULDBLOCK: + if MAYBE_EWOULDBLOCK is not None and errno == MAYBE_EWOULDBLOCK: return True return False diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py new file mode 100644 --- /dev/null +++ b/pypy/module/_file/readinto.py @@ -0,0 +1,81 @@ +import sys, errno +from rpython.rlib import rposix +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.module._file.interp_file import is_wouldblock_error, signal_checker + +_WIN32 = sys.platform.startswith('win') +UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' + +os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read', + [rffi.INT, rffi.CCHARP, rffi.SIZE_T], + rffi.SIZE_T, save_err=rffi.RFFI_SAVE_ERRNO) + + +def direct_readinto(self, w_rwbuffer): + rwbuffer = self.space.writebuf_w(w_rwbuffer) + stream = self.getstream() + size = rwbuffer.getlength() + target_address = lltype.nullptr(rffi.CCHARP.TO) + fd = -1 + target_pos = 0 + + if size > 64: + try: + target_address = rwbuffer.get_raw_address() + except ValueError: + pass + else: + fd = stream.try_to_find_file_descriptor() + + if fd < 0 or not target_address: + # fall-back + MAX_PART = 1024 * 1024 # 1 MB + while size > MAX_PART: + data = self.direct_read(MAX_PART) + rwbuffer.setslice(target_pos, data) + target_pos += len(data) + size -= len(data) + if len(data) != MAX_PART: + break + else: + data = self.direct_read(size) + rwbuffer.setslice(target_pos, data) + target_pos += len(data) + + else: + # optimized case: reading more than 64 bytes into a rwbuffer + # with a valid raw address + self.check_readable() + + # first "read" the part that is already sitting in buffers, if any + initial_size = min(size, stream.count_buffered_bytes()) + if initial_size > 0: + data = stream.read(initial_size) + rwbuffer.setslice(target_pos, data) + target_pos += len(data) + size -= len(data) + + # then call os_read() to get the rest + if size > 0: + stream.flush() + while True: + got = os_read(fd, rffi.ptradd(target_address, target_pos), size) + if got > 0: + target_pos += got + size -= got + if size <= 0: + break + elif got == 0: + break + else: + err = rposix.get_saved_errno() + if err == errno.EINTR: + signal_checker(self.space)() + continue + if is_wouldblock_error(err) and target_pos > 0: + break + raise OSError(err, "read error") + keepalive_until_here(rwbuffer) + + return self.space.wrap(target_pos) diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py --- a/pypy/module/_vmprof/__init__.py +++ b/pypy/module/_vmprof/__init__.py @@ -2,7 +2,7 @@ class Module(MixedModule): """ - Write me :) + VMProf for PyPy: a statistical profiler """ appleveldefs = { } @@ -10,9 +10,13 @@ interpleveldefs = { 'enable': 'interp_vmprof.enable', 'disable': 'interp_vmprof.disable', + 'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError', } - def setup_after_space_initialization(self): - # force the __extend__ hacks to occur early - from pypy.module._vmprof.interp_vmprof import VMProf - self.vmprof = VMProf() + +# Force the __extend__ hacks and method replacements to occur +# early. Without this, for example, 'PyCode._init_ready' was +# already found by the annotator to be the original empty +# method, and the annotator doesn't notice that interp_vmprof.py +# (loaded later) replaces this method. +import pypy.module._vmprof.interp_vmprof diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -1,252 +1,74 @@ -import py, os, sys -from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance -from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib import jit, rposix, rgc -from rpython.rlib.rarithmetic import ovfcheck_float_to_int -from rpython.rtyper.tool import rffi_platform as platform -from rpython.rlib.rstring import StringBuilder -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import oefmt, wrap_oserror, OperationError +from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pycode import PyCode +from pypy.interpreter.baseobjspace import W_Root +from rpython.rlib import rvmprof -ROOT = py.path.local(__file__).join('..') -SRC = ROOT.join('src') +# ____________________________________________________________ -# by default, we statically link vmprof.c into pypy; however, if you set -# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so -# which is expected to be inside pypy/module/_vmprof/src: this is very useful -# during development. Note that you have to manually build libvmprof by -# running make inside the src dir -DYNAMIC_VMPROF = False -if sys.platform.startswith('linux'): - libs = ['dl'] -else: - libs = [] +_get_code = lambda frame, w_inputvalue, operr: frame.pycode +_decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root) +my_execute_frame = _decorator(PyFrame.execute_frame) -eci_kwds = dict( - include_dirs = [SRC], - includes = ['vmprof.h', 'trampoline.h'], - separate_module_files = [SRC.join('trampoline.vmprof.s')], - libraries = libs, - - post_include_bits=[""" - int pypy_vmprof_init(void); - """], - - separate_module_sources=[""" - int pypy_vmprof_init(void) { - return vmprof_set_mainloop(pypy_execute_frame_trampoline, 0, - NULL); - } - """], - ) - - -if DYNAMIC_VMPROF: - eci_kwds['libraries'] += ['vmprof'] - eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC] -else: - eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')] - -eci = ExternalCompilationInfo(**eci_kwds) - -check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[ - SRC.join('fake_pypy_api.c')])) - -platform.verify_eci(check_eci) - -pypy_execute_frame_trampoline = rffi.llexternal( - "pypy_execute_frame_trampoline", - [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed], - llmemory.GCREF, - compilation_info=eci, - _nowrapper=True, sandboxsafe=True, - random_effects_on_gcobjs=True) - -pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], rffi.INT, - compilation_info=eci) -vmprof_enable = rffi.llexternal("vmprof_enable", - [rffi.INT, rffi.LONG, rffi.INT, - rffi.CCHARP, rffi.INT], - rffi.INT, compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) -vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, - compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) -vmprof_get_error = rffi.llexternal("vmprof_get_error", [], rffi.CCHARP, - compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) - -vmprof_register_virtual_function = rffi.llexternal( - "vmprof_register_virtual_function", - [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, - compilation_info=eci, _nowrapper=True) - -original_execute_frame = PyFrame.execute_frame.im_func -original_execute_frame.c_name = 'pypy_pyframe_execute_frame' -original_execute_frame._dont_inline_ = True class __extend__(PyFrame): - def execute_frame(frame, w_inputvalue=None, operr=None): - # go through the asm trampoline ONLY if we are translated but not being JITted. - # - # If we are not translated, we obviously don't want to go through the - # trampoline because there is no C function it can call. - # - # If we are being JITted, we want to skip the trampoline, else the JIT - # cannot see throug it - if we_are_translated() and not jit.we_are_jitted(): - # if we are translated, call the trampoline - gc_frame = cast_instance_to_gcref(frame) - gc_inputvalue = cast_instance_to_gcref(w_inputvalue) - gc_operr = cast_instance_to_gcref(operr) - unique_id = frame.pycode._unique_id - gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue, - gc_operr, unique_id) - return cast_base_ptr_to_instance(W_Root, gc_result) - else: - return original_execute_frame(frame, w_inputvalue, operr) + def execute_frame(self, w_inputvalue=None, operr=None): + # indirection for the optional arguments + return my_execute_frame(self, w_inputvalue, operr) +def _safe(s): + if len(s) > 110: + s = s[:107] + '...' + return s.replace(':', ';') -def write_long_to_string_builder(l, b): - if sys.maxint == 2147483647: - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) - else: - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) - b.append(chr((l >> 32) & 0xff)) - b.append(chr((l >> 40) & 0xff)) - b.append(chr((l >> 48) & 0xff)) - b.append(chr((l >> 56) & 0xff)) +def _get_full_name(pycode): + # careful, must not have extraneous ':' or be longer than 255 chars + return "py:%s:%d:%s" % (_safe(pycode.co_name), pycode.co_firstlineno, + _safe(pycode.co_filename)) -def try_cast_to_pycode(gcref): - return rgc.try_cast_gcref_to_instance(PyCode, gcref) +rvmprof.register_code_object_class(PyCode, _get_full_name) -MAX_CODES = 1000 -class VMProf(object): - def __init__(self): - self.is_enabled = False - self.ever_enabled = False - self.fileno = -1 - self.current_codes = [] +def _init_ready(pycode): + rvmprof.register_code(pycode, _get_full_name) - def enable(self, space, fileno, period_usec): - if self.is_enabled: - raise oefmt(space.w_ValueError, "_vmprof already enabled") - self.fileno = fileno - self.is_enabled = True - self.write_header(fileno, period_usec) - if not self.ever_enabled: - if we_are_translated(): - res = pypy_vmprof_init() - if res: - raise OperationError( - space.w_IOError, - space.wrap(rffi.charp2str(vmprof_get_error()))) - self.ever_enabled = True - self.gather_all_code_objs(space) - space.register_code_callback(vmprof_register_code) - if we_are_translated(): - # does not work untranslated - res = vmprof_enable(fileno, period_usec, 0, - lltype.nullptr(rffi.CCHARP.TO), 0) - else: - res = 0 - if res == -1: - raise wrap_oserror(space, OSError(rposix.get_saved_errno(), - "_vmprof.enable")) +PyCode._init_ready = _init_ready - def gather_all_code_objs(self, space): - all_code_objs = rgc.do_get_objects(try_cast_to_pycode) - for code in all_code_objs: - self.register_code(space, code) - def write_header(self, fileno, period_usec): - assert period_usec > 0 - b = StringBuilder() - write_long_to_string_builder(0, b) - write_long_to_string_builder(3, b) - write_long_to_string_builder(0, b) - write_long_to_string_builder(period_usec, b) - write_long_to_string_builder(0, b) - b.append('\x04') # interp name - b.append(chr(len('pypy'))) - b.append('pypy') - os.write(fileno, b.build()) +# ____________________________________________________________ - def register_code(self, space, code): - if self.fileno == -1: - raise OperationError(space.w_RuntimeError, - space.wrap("vmprof not running")) - self.current_codes.append(code) - if len(self.current_codes) >= MAX_CODES: - self._flush_codes(space) - def _flush_codes(self, space): - b = StringBuilder() - for code in self.current_codes: - name = code._get_full_name() - b.append('\x02') - write_long_to_string_builder(code._unique_id, b) - write_long_to_string_builder(len(name), b) - b.append(name) - os.write(self.fileno, b.build()) - self.current_codes = [] +class Cache: + def __init__(self, space): + self.w_VMProfError = space.new_exception_class("_vmprof.VMProfError") - def disable(self, space): - if not self.is_enabled: - raise oefmt(space.w_ValueError, "_vmprof not enabled") - self.is_enabled = False - space.register_code_callback(None) - self._flush_codes(space) - self.fileno = -1 - if we_are_translated(): - # does not work untranslated - res = vmprof_disable() - else: - res = 0 - if res == -1: - raise wrap_oserror(space, OSError(rposix.get_saved_errno(), - "_vmprof.disable")) +def VMProfError(space, e): + w_VMProfError = space.fromcache(Cache).w_VMProfError + return OperationError(w_VMProfError, space.wrap(e.msg)) -def vmprof_register_code(space, code): - from pypy.module._vmprof import Module - mod_vmprof = space.getbuiltinmodule('_vmprof') - assert isinstance(mod_vmprof, Module) - mod_vmprof.vmprof.register_code(space, code) @unwrap_spec(fileno=int, period=float) -def enable(space, fileno, period=0.01): # default 100 Hz - from pypy.module._vmprof import Module - mod_vmprof = space.getbuiltinmodule('_vmprof') - assert isinstance(mod_vmprof, Module) - # +def enable(space, fileno, period): + """Enable vmprof. Writes go to the given 'fileno', a file descriptor + opened for writing. *The file descriptor must remain open at least + until disable() is called.* + + 'interval' is a float representing the sampling interval, in seconds. + Must be smaller than 1.0 + """ try: - period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5) - if period_usec <= 0 or period_usec >= 1e6: - # we don't want seconds here at all - raise ValueError - except (ValueError, OverflowError): - raise OperationError(space.w_ValueError, - space.wrap("'period' too large or non positive")) - # - mod_vmprof.vmprof.enable(space, fileno, period_usec) + rvmprof.enable(fileno, period) + except rvmprof.VMProfError, e: + raise VMProfError(space, e) def disable(space): - from pypy.module._vmprof import Module - mod_vmprof = space.getbuiltinmodule('_vmprof') - assert isinstance(mod_vmprof, Module) - mod_vmprof.vmprof.disable(space) - + """Disable vmprof. Remember to close the file descriptor afterwards + if necessary. + """ + try: + rvmprof.disable() + except rvmprof.VMProfError, e: + raise VMProfError(space, e) diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h deleted file mode 100644 --- a/pypy/module/_vmprof/src/config.h +++ /dev/null @@ -1,6 +0,0 @@ -#define HAVE_SYS_UCONTEXT_H -#if defined(__FreeBSD__) || defined(__APPLE__) -#define PC_FROM_UCONTEXT uc_mcontext.mc_rip -#else -#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP] -#endif diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c deleted file mode 100644 --- a/pypy/module/_vmprof/src/fake_pypy_api.c +++ /dev/null @@ -1,4 +0,0 @@ - -void pypy_pyframe_execute_frame(void) -{ -} diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c deleted file mode 100644 --- a/pypy/module/_vmprof/src/get_custom_offset.c +++ /dev/null @@ -1,80 +0,0 @@ - -#ifdef PYPY_JIT_CODEMAP - -extern volatile int pypy_codemap_currently_invalid; - -void *pypy_find_codemap_at_addr(long addr, long *start_addr); -long pypy_yield_codemap_at_addr(void *codemap_raw, long addr, - long *current_pos_addr); -long pypy_jit_stack_depth_at_loc(long loc); - -#endif - - -void vmprof_set_tramp_range(void* start, void* end) -{ -} - -int custom_sanity_check() -{ -#ifdef PYPY_JIT_CODEMAP - return !pypy_codemap_currently_invalid; -#else - return 1; -#endif -} - -static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) { -#ifdef PYPY_JIT_CODEMAP - intptr_t ip_l = (intptr_t)ip; - return pypy_jit_stack_depth_at_loc(ip_l); -#else - return 0; -#endif -} - -static long vmprof_write_header_for_jit_addr(void **result, long n, - void *ip, int max_depth) -{ -#ifdef PYPY_JIT_CODEMAP - void *codemap; - long current_pos = 0; - intptr_t id; - long start_addr = 0; - intptr_t addr = (intptr_t)ip; - int start, k; - void *tmp; - - codemap = pypy_find_codemap_at_addr(addr, &start_addr); - if (codemap == NULL) - // not a jit code at all - return n; - - // modify the last entry to point to start address and not the random one - // in the middle - result[n - 1] = (void*)start_addr; - result[n] = (void*)2; - n++; - start = n; - while (n < max_depth) { - id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos); - if (id == -1) - // finish - break; - if (id == 0) - continue; // not main codemap - result[n++] = (void *)id; - } - k = 0; - while (k < (n - start) / 2) { - tmp = result[start + k]; - result[start + k] = result[n - k - 1]; - result[n - k - 1] = tmp; - k++; - } - if (n < max_depth) { - result[n++] = (void*)3; - } -#endif - return n; -} diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h deleted file mode 100644 --- a/pypy/module/_vmprof/src/trampoline.h +++ /dev/null @@ -1,1 +0,0 @@ -void* pypy_execute_frame_trampoline(void*, void*, void*, long); diff --git a/pypy/module/_vmprof/src/trampoline.vmprof.s b/pypy/module/_vmprof/src/trampoline.vmprof.s deleted file mode 100644 --- a/pypy/module/_vmprof/src/trampoline.vmprof.s +++ /dev/null @@ -1,15 +0,0 @@ -// NOTE: you need to use TABs, not spaces! - - .text - .globl pypy_execute_frame_trampoline - .type pypy_execute_frame_trampoline, @function -pypy_execute_frame_trampoline: - .cfi_startproc - pushq %rcx - .cfi_def_cfa_offset 16 - call pypy_pyframe_execute_frame at PLT - popq %rcx - .cfi_def_cfa_offset 8 - ret - .cfi_endproc - .size pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c deleted file mode 100644 --- a/pypy/module/_vmprof/src/vmprof.c +++ /dev/null @@ -1,463 +0,0 @@ -/* VMPROF - * - * statistical sampling profiler specifically designed to profile programs - * which run on a Virtual Machine and/or bytecode interpreter, such as Python, - * etc. - * - * The logic to dump the C stack traces is partly stolen from the code in gperftools. - * The file "getpc.h" has been entirely copied from gperftools. - * - * Tested only on gcc, linux, x86_64. - * - * Copyright (C) 2014-2015 - * Antonio Cuni - anto.cuni at gmail.com - * Maciej Fijalkowski - fijall at gmail.com - * - */ - - -#include "getpc.h" // should be first to get the _GNU_SOURCE dfn -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -//#define UNW_LOCAL_ONLY -//#include - -#include "vmprof.h" -#if defined(__FreeBSD__) || defined(__APPLE__) -#define sighandler_t sig_t -#endif - -#define _unused(x) ((void)x) - -#define MAX_FUNC_NAME 128 -#define MAX_STACK_DEPTH 1024 -#define BUFFER_SIZE 8192 - - -static int profile_file = 0; -static char profile_write_buffer[BUFFER_SIZE]; -static int profile_buffer_position = 0; -void* vmprof_mainloop_func; -char* vmprof_error = NULL; -static ptrdiff_t mainloop_sp_offset; -static vmprof_get_virtual_ip_t mainloop_get_virtual_ip; -static long last_period_usec = 0; -static int atfork_hook_installed = 0; - - -/* ************************************************************* - * functions to write a profile file compatible with gperftools - * ************************************************************* - */ - -#define MARKER_STACKTRACE '\x01' -#define MARKER_VIRTUAL_IP '\x02' -#define MARKER_TRAILER '\x03' - -int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL; -int (*unw_step)(unw_cursor_t*) = NULL; -int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; -int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; - -static void prof_word(long x) { - ((long*)(profile_write_buffer + profile_buffer_position))[0] = x; - profile_buffer_position += sizeof(long); -} - -static void prof_header(long period_usec) { - // XXX never used here? - prof_word(0); - prof_word(3); - prof_word(0); - prof_word(period_usec); - prof_word(0); - write(profile_file, profile_write_buffer, profile_buffer_position); - profile_buffer_position = 0; -} - -static void prof_write_stacktrace(void** stack, int depth, int count) { - int i; - char marker = MARKER_STACKTRACE; - - profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE; - prof_word(count); - prof_word(depth); - for(i=0; isp = bp; - bp -= sizeof(void*); - cp2->ip = ((void**)bp)[0]; - // the ret is on the top of the stack minus WORD - return 1; - } -} - - -/* ************************************************************* - * functions to dump the stack trace - * ************************************************************* - */ - -// The original code here has a comment, "stolen from pprof", -// about a "__thread int recursive". But general __thread -// variables are not really supposed to be accessed from a -// signal handler. Moreover, we are using SIGPROF, which -// should not be recursively called on the same thread. -//static __thread int recursive; - -int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) { - void *ip; - int n = 0; - unw_cursor_t cursor; - unw_context_t uc = *ucontext; - //if (recursive) { - // return 0; - //} - if (!custom_sanity_check()) { - return 0; - } - //++recursive; - - int ret = unw_init_local(&cursor, &uc); - assert(ret >= 0); - _unused(ret); - int first_run = 1; - - while (n < max_depth) { - if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { - break; - } - - unw_proc_info_t pip; - unw_get_proc_info(&cursor, &pip); - - /* char funcname[4096]; */ - /* unw_word_t offset; */ - /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */ - /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */ - - /* if n==0, it means that the signal handler interrupted us while we - were in the trampoline, so we are not executing (yet) the real main - loop function; just skip it */ - if (vmprof_mainloop_func && - (void*)pip.start_ip == (void*)vmprof_mainloop_func && - n > 0) { - // found main loop stack frame - void* sp; - unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp); - void *arg_addr = (char*)sp + mainloop_sp_offset; - void **arg_ptr = (void**)arg_addr; - // fprintf(stderr, "stacktrace mainloop: rsp %p &f2 %p offset %ld\n", - // sp, arg_addr, mainloop_sp_offset); - if (mainloop_get_virtual_ip) { - ip = mainloop_get_virtual_ip(*arg_ptr); - } else { - ip = *arg_ptr; - } - } - - result[n++] = ip; - n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth); - if (vmprof_unw_step(&cursor, first_run) <= 0) { - break; - } - first_run = 0; - } - //--recursive; - return n; -} - - -static int __attribute__((noinline)) frame_forcer(int rv) { - return rv; -} - -static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) { - void* stack[MAX_STACK_DEPTH]; - int saved_errno = errno; - stack[0] = GetPC((ucontext_t*)ucontext); - int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext)); - depth++; // To account for pc value in stack[0]; - prof_write_stacktrace(stack, depth, 1); - errno = saved_errno; -} - -/* ************************************************************* - * functions to enable/disable the profiler - * ************************************************************* - */ - -static int open_profile(int fd, long period_usec, int write_header, char *s, - int slen) { - if ((fd = dup(fd)) == -1) { - return -1; - } - profile_buffer_position = 0; - profile_file = fd; - if (write_header) - prof_header(period_usec); - if (s) - write(profile_file, s, slen); - return 0; -} - -static int close_profile(void) { - // XXX all of this can happily fail - FILE* src; - char buf[BUFSIZ]; - size_t size; - int marker = MARKER_TRAILER; - write(profile_file, &marker, 1); - -#ifdef __linux__ - // copy /proc/PID/maps to the end of the profile file - sprintf(buf, "/proc/%d/maps", getpid()); - src = fopen(buf, "r"); - if (!src) { - vmprof_error = "error opening proc maps"; - return -1; - } - while ((size = fread(buf, 1, BUFSIZ, src))) { - write(profile_file, buf, size); - } - fclose(src); -#else - // freebsd and mac - sprintf(buf, "procstat -v %d", getpid()); - src = popen(buf, "r"); - if (!src) { - vmprof_error = "error calling procstat"; - return -1; - } - while ((size = fread(buf, 1, BUFSIZ, src))) { - write(profile_file, buf, size); - } - pclose(src); -#endif - close(profile_file); - return 0; -} - - -static int install_sigprof_handler(void) { - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_sigaction = sigprof_handler; - sa.sa_flags = SA_RESTART | SA_SIGINFO; - if (sigemptyset(&sa.sa_mask) == -1 || - sigaction(SIGPROF, &sa, NULL) == -1) { - return -1; - } - return 0; -} - -static int remove_sigprof_handler(void) { - sighandler_t res = signal(SIGPROF, SIG_DFL); - if (res == SIG_ERR) { - return -1; - } - return 0; -}; - -static int install_sigprof_timer(long period_usec) { - static struct itimerval timer; - last_period_usec = period_usec; - timer.it_interval.tv_sec = 0; - timer.it_interval.tv_usec = period_usec; - timer.it_value = timer.it_interval; - if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { - return -1; - } - return 0; -} - -static int remove_sigprof_timer(void) { - static struct itimerval timer; - timer.it_interval.tv_sec = 0; - timer.it_interval.tv_usec = 0; - timer.it_value.tv_sec = 0; - timer.it_value.tv_usec = 0; - if (setitimer(ITIMER_PROF, &timer, NULL) != 0) { - return -1; - } - return 0; -} - -static void atfork_disable_timer(void) { - if (last_period_usec) { - remove_sigprof_timer(); - } -} - -static void atfork_enable_timer(void) { - if (last_period_usec) { - install_sigprof_timer(last_period_usec); - } -} - -static int install_pthread_atfork_hooks(void) { - /* this is needed to prevent the problems described there: - - http://code.google.com/p/gperftools/issues/detail?id=278 - - http://lists.debian.org/debian-glibc/2010/03/msg00161.html - - TL;DR: if the RSS of the process is large enough, the clone() syscall - will be interrupted by the SIGPROF before it can complete, then - retried, interrupted again and so on, in an endless loop. The - solution is to disable the timer around the fork, and re-enable it - only inside the parent. - */ - if (atfork_hook_installed) - return 0; - int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); - if (ret != 0) - return -1; - atfork_hook_installed = 1; - return 0; -} - -/* ************************************************************* - * public API - * ************************************************************* - */ - -int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, - vmprof_get_virtual_ip_t get_virtual_ip) { - void *libhandle; - - mainloop_sp_offset = sp_offset; - mainloop_get_virtual_ip = get_virtual_ip; - vmprof_mainloop_func = func; - if (!unw_get_reg) { - if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) { - vmprof_error = dlerror(); - return -1; - } - if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) { - vmprof_error = dlerror(); - return -1; - } - if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))){ - vmprof_error = dlerror(); - return -1; - } - if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) { - vmprof_error = dlerror(); - return -1; - } - if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) { - vmprof_error = dlerror(); - return -1; - } - } - return 0; -} - -char* vmprof_get_error() -{ - char* res; - res = vmprof_error; - vmprof_error = NULL; - return res; -} - -int vmprof_enable(int fd, long period_usec, int write_header, char *s, - int slen) -{ - assert(period_usec > 0); - if (open_profile(fd, period_usec, write_header, s, slen) == -1) { - return -1; - } - if (install_sigprof_handler() == -1) { - return -1; - } - if (install_sigprof_timer(period_usec) == -1) { - return -1; - } - if (install_pthread_atfork_hooks() == -1) { - return -1; - } - return 0; -} - -int vmprof_disable(void) { - if (remove_sigprof_timer() == -1) { - return -1; - } - last_period_usec = 0; - if (remove_sigprof_handler() == -1) { - return -1; - } - if (close_profile() == -1) { - return -1; - } - return 0; -} - -void vmprof_register_virtual_function(const char* name, void* start, void* end) { - // XXX unused by pypy - // for now *end is simply ignored - char buf[1024]; - int lgt = strlen(name) + 2 * sizeof(long) + 1; - - if (lgt > 1024) { - lgt = 1024; - } - buf[0] = MARKER_VIRTUAL_IP; - ((void **)(((void*)buf) + 1))[0] = start; - ((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1; - strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1); - write(profile_file, buf, lgt); -} diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h deleted file mode 100644 --- a/pypy/module/_vmprof/src/vmprof.h +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef VMPROF_VMPROF_H_ -#define VMPROF_VMPROF_H_ - -#include -#include -#include - -// copied from libunwind.h - -typedef enum - { - UNW_X86_64_RAX, - UNW_X86_64_RDX, - UNW_X86_64_RCX, - UNW_X86_64_RBX, - UNW_X86_64_RSI, - UNW_X86_64_RDI, - UNW_X86_64_RBP, - UNW_X86_64_RSP, - UNW_X86_64_R8, - UNW_X86_64_R9, - UNW_X86_64_R10, - UNW_X86_64_R11, - UNW_X86_64_R12, - UNW_X86_64_R13, - UNW_X86_64_R14, - UNW_X86_64_R15, - UNW_X86_64_RIP, -#ifdef CONFIG_MSABI_SUPPORT - UNW_X86_64_XMM0, - UNW_X86_64_XMM1, - UNW_X86_64_XMM2, - UNW_X86_64_XMM3, - UNW_X86_64_XMM4, - UNW_X86_64_XMM5, - UNW_X86_64_XMM6, - UNW_X86_64_XMM7, - UNW_X86_64_XMM8, - UNW_X86_64_XMM9, - UNW_X86_64_XMM10, - UNW_X86_64_XMM11, - UNW_X86_64_XMM12, - UNW_X86_64_XMM13, - UNW_X86_64_XMM14, - UNW_X86_64_XMM15, - UNW_TDEP_LAST_REG = UNW_X86_64_XMM15, -#else - UNW_TDEP_LAST_REG = UNW_X86_64_RIP, -#endif - - /* XXX Add other regs here */ - - /* frame info (read-only) */ - UNW_X86_64_CFA, - - UNW_TDEP_IP = UNW_X86_64_RIP, - UNW_TDEP_SP = UNW_X86_64_RSP, - UNW_TDEP_BP = UNW_X86_64_RBP, - UNW_TDEP_EH = UNW_X86_64_RAX - } -x86_64_regnum_t; - -typedef uint64_t unw_word_t; - -#define UNW_TDEP_CURSOR_LEN 127 - -typedef struct unw_cursor - { - unw_word_t opaque[UNW_TDEP_CURSOR_LEN]; - } -unw_cursor_t; - -#define UNW_REG_IP UNW_X86_64_RIP -#define UNW_REG_SP UNW_X86_64_RSP - -typedef ucontext_t unw_context_t; - -typedef struct unw_proc_info - { - unw_word_t start_ip; /* first IP covered by this procedure */ - unw_word_t end_ip; /* first IP NOT covered by this procedure */ - unw_word_t lsda; /* address of lang.-spec. data area (if any) */ - unw_word_t handler; /* optional personality routine */ - unw_word_t gp; /* global-pointer value for this procedure */ - unw_word_t flags; /* misc. flags */ - - int format; /* unwind-info format (arch-specific) */ - int unwind_info_size; /* size of the information (if applicable) */ - void *unwind_info; /* unwind-info (arch-specific) */ - } -unw_proc_info_t; - -// functions copied from libunwind using dlopen - -extern int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*); -extern int (*unw_step)(unw_cursor_t*); -extern int (*unw_init_local)(unw_cursor_t *, unw_context_t *); -extern int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *); - -// end of copy - -extern char* vmprof_error; - -typedef void* (*vmprof_get_virtual_ip_t)(void*); -char* vmprof_get_error(); - -extern void* vmprof_mainloop_func; -int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, - vmprof_get_virtual_ip_t get_virtual_ip); - -void vmprof_register_virtual_function(const char* name, void* start, void* end); - - -int vmprof_enable(int fd, long period_usec, int write_header, char* vips, - int vips_len); -int vmprof_disable(void); - -// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library) -void vmprof_set_tramp_range(void* start, void* end); - -#endif diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -1,14 +1,14 @@ -import tempfile +from rpython.tool.udir import udir from pypy.tool.pytest.objspace import gettestobjspace class AppTestVMProf(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) - cls.tmpfile = tempfile.NamedTemporaryFile() + cls.tmpfile = udir.join('test__vmprof.1').open('wb') cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno()) cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name) - cls.tmpfile2 = tempfile.NamedTemporaryFile() + cls.tmpfile2 = udir.join('test__vmprof.2').open('wb') cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno()) cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name) @@ -29,19 +29,23 @@ while i < len(s): if s[i] == '\x03': break - if s[i] == '\x01': - xxx - assert s[i] == '\x02' - i += 1 - _, size = struct.unpack("ll", s[i:i + 2 * WORD]) - count += 1 - i += 2 * WORD + size + elif s[i] == '\x01': + i += 1 + _, size = struct.unpack("ll", s[i:i + 2 * WORD]) + i += 2 * WORD + size * struct.calcsize("P") + elif s[i] == '\x02': + i += 1 + _, size = struct.unpack("ll", s[i:i + 2 * WORD]) + count += 1 + i += 2 * WORD + size + else: + raise AssertionError(ord(s[i])) return count import _vmprof - _vmprof.enable(self.tmpfileno) + _vmprof.enable(self.tmpfileno, 0.01) _vmprof.disable() - s = open(self.tmpfilename).read() + s = open(self.tmpfilename, 'rb').read() no_of_codes = count(s) assert no_of_codes > 10 d = {} @@ -50,14 +54,14 @@ pass """ in d - _vmprof.enable(self.tmpfileno2) + _vmprof.enable(self.tmpfileno2, 0.01) exec """def foo2(): pass """ in d _vmprof.disable() - s = open(self.tmpfilename2).read() + s = open(self.tmpfilename2, 'rb').read() no_of_codes2 = count(s) assert "py:foo:" in s assert "py:foo2:" in s @@ -65,8 +69,9 @@ def test_enable_ovf(self): import _vmprof - raises(ValueError, _vmprof.enable, 999, 0) - raises(ValueError, _vmprof.enable, 999, -2.5) - raises(ValueError, _vmprof.enable, 999, 1e300) - raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300) - raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300)) + raises(_vmprof.VMProfError, _vmprof.enable, 999, 0) + raises(_vmprof.VMProfError, _vmprof.enable, 999, -2.5) + raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300) + raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300 * 1e300) + NaN = (1e300*1e300) / (1e300*1e300) + raises(_vmprof.VMProfError, _vmprof.enable, 999, NaN) diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py --- a/pypy/module/_vmprof/test/test_direct.py +++ b/pypy/module/_vmprof/test/test_direct.py @@ -5,7 +5,8 @@ except ImportError: py.test.skip('cffi required') -srcdir = py.path.local(__file__).join("..", "..", "src") +from rpython.rlib import rvmprof +srcdir = py.path.local(rvmprof.__file__).join("..", "src") ffi = cffi.FFI() ffi.cdef(""" @@ -17,6 +18,8 @@ """) lib = ffi.verify(""" +#define PYPY_JIT_CODEMAP + volatile int pypy_codemap_currently_invalid = 0; long buffer[] = {0, 0, 0, 0, 0}; @@ -39,7 +42,7 @@ } -""" + open(str(srcdir.join("get_custom_offset.c"))).read()) +""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read()) class TestDirect(object): def test_infrastructure(self): diff --git a/pypy/module/cpyext/TODO b/pypy/module/cpyext/TODO deleted file mode 100644 --- a/pypy/module/cpyext/TODO +++ /dev/null @@ -1,26 +0,0 @@ - - Complete the PyTypeObject initialization code. (see XXX in the code) - - Implement further method callers. - - Copy the slots from the base. - - Those tasks are necessary to be able to call slots from C code correctly. - - Additionally, implement further slot wrappers. This is necessary to call - slots of PTOs defined in C. - - Complete the Members support. - - - Use a WeakKeyDictionary to count how often a PyObject is allocated for - a given wrapped object and use this to assess whether optimizations are - useful - - - replace @cpython_api(external=False) by another explicit name: all - it does is a lltype function pointer, no C code involved. - - - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads). - - - refactor management of py_objects_r2w and py_objects_w2r, this can - probably be expressed in terms of _PyObject_GC_TRACK macros. - - - PyWeakref_GetObject() returns a borrowed reference, but this turns the - WeakRef into a strong reference! - - - sort out pypy's buffer protocol. PyPy's buffer right now don't support - raw memory (except array which supports it in a hackish way), which - should be fixed in order to make it nicely work with cpyext. diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py --- a/pypy/module/micronumpy/casting.py +++ b/pypy/module/micronumpy/casting.py @@ -145,23 +145,32 @@ # equivalent to PyArray_CanCastTypeTo if origin == target: return True - if origin.is_record() or target.is_record(): - return can_cast_record(space, origin, target, casting) + if casting == 'unsafe': + return True + elif casting == 'no': + return origin.eq(space, target) + if origin.num == target.num: + if origin.is_record(): + return (target.is_record() and + can_cast_record(space, origin, target, casting)) + else: + if casting == 'equiv': + return origin.elsize == target.elsize + elif casting == 'safe': + return origin.elsize <= target.elsize + else: + return True - if casting == 'no': - return origin.eq(space, target) - elif casting == 'equiv': - return origin.num == target.num and origin.elsize == target.elsize - elif casting == 'unsafe': - return True elif casting == 'same_kind': if can_cast_to(origin, target): return True if origin.kind in kind_ordering and target.kind in kind_ordering: return kind_ordering[origin.kind] <= kind_ordering[target.kind] return False - else: # 'safe' + elif casting == 'safe': return can_cast_to(origin, target) + else: # 'equiv' + return origin.num == target.num and origin.elsize == target.elsize def can_cast_record(space, origin, target, casting): if origin is target: diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -101,6 +101,9 @@ @specialize.argtype(1) def box(self, value): + if self.is_record(): + raise oefmt(self.itemtype.space.w_NotImplementedError, + "cannot box a value into a 'record' dtype, this is a bug please report it") return self.itemtype.box(value) @specialize.argtype(1, 2) @@ -1028,6 +1031,11 @@ elif space.isinstance_w(w_dtype, space.w_tuple): w_dtype0 = space.getitem(w_dtype, space.wrap(0)) w_dtype1 = space.getitem(w_dtype, space.wrap(1)) + if space.isinstance_w(w_dtype0, space.w_type) and \ + space.isinstance_w(w_dtype1, space.w_list): + #obscure api - (subclass, spec). Ignore the subclass + return make_new_dtype(space, w_subtype, w_dtype1, alignment, + copy=copy, w_shape=w_shape, w_metadata=w_metadata) subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy) assert isinstance(subdtype, W_Dtype) if subdtype.elsize == 0: diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py --- a/pypy/module/micronumpy/iterators.py +++ b/pypy/module/micronumpy/iterators.py @@ -204,17 +204,16 @@ self.array.setitem(state.offset, elem) -def AxisIter(array, shape, axis, cumulative): +def AxisIter(array, shape, axis): strides = array.get_strides() backstrides = array.get_backstrides() - if not cumulative: - if len(shape) == len(strides): - # keepdims = True - strides = strides[:axis] + [0] + strides[axis + 1:] - backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:] - else: - strides = strides[:axis] + [0] + strides[axis:] - backstrides = backstrides[:axis] + [0] + backstrides[axis:] + if len(shape) == len(strides): + # keepdims = True + strides = strides[:axis] + [0] + strides[axis + 1:] + backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:] + else: + strides = strides[:axis] + [0] + strides[axis:] + backstrides = backstrides[:axis] + [0] + backstrides[axis:] return ArrayIter(array, support.product(shape), shape, strides, backstrides) diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -9,7 +9,7 @@ from pypy.module.micronumpy import support, constants as NPY from pypy.module.micronumpy.base import W_NDimArray, convert_to_array from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \ - AllButAxisIter + AllButAxisIter, ArrayIter from pypy.interpreter.argument import Arguments @@ -190,23 +190,64 @@ source_state = source_iter.next(source_state) return target -reduce_driver = jit.JitDriver(name='numpy_reduce', - greens = ['shapelen', 'func', 'done_func', - 'calc_dtype'], - reds = 'auto') -def compute_reduce(space, obj, calc_dtype, func, done_func, identity): - obj_iter, obj_state = obj.create_iter() +def split_iter(arr, axis_flags): + """Prepare 2 iterators for nested iteration over `arr`. + + Arguments: + arr: instance of BaseConcreteArray + axis_flags: list of bools, one for each dimension of `arr`.The inner + iterator operates over the dimensions for which the flag is True + """ + shape = arr.get_shape() + strides = arr.get_strides() + backstrides = arr.get_backstrides() + shapelen = len(shape) + assert len(axis_flags) == shapelen + inner_shape = [-1] * shapelen + inner_strides = [-1] * shapelen + inner_backstrides = [-1] * shapelen + outer_shape = [-1] * shapelen + outer_strides = [-1] * shapelen + outer_backstrides = [-1] * shapelen + for i in range(len(shape)): + if axis_flags[i]: + inner_shape[i] = shape[i] + inner_strides[i] = strides[i] + inner_backstrides[i] = backstrides[i] + outer_shape[i] = 1 + outer_strides[i] = 0 + outer_backstrides[i] = 0 + else: + outer_shape[i] = shape[i] + outer_strides[i] = strides[i] + outer_backstrides[i] = backstrides[i] + inner_shape[i] = 1 + inner_strides[i] = 0 + inner_backstrides[i] = 0 + inner_iter = ArrayIter(arr, support.product(inner_shape), + inner_shape, inner_strides, inner_backstrides) + outer_iter = ArrayIter(arr, support.product(outer_shape), + outer_shape, outer_strides, outer_backstrides) + return inner_iter, outer_iter + + +reduce_flat_driver = jit.JitDriver( + name='numpy_reduce_flat', + greens = ['shapelen', 'func', 'done_func', 'calc_dtype'], reds = 'auto') + +def reduce_flat(space, func, w_arr, calc_dtype, done_func, identity): + obj_iter, obj_state = w_arr.create_iter() if identity is None: cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) obj_state = obj_iter.next(obj_state) else: cur_value = identity.convert_to(space, calc_dtype) - shapelen = len(obj.get_shape()) + shapelen = len(w_arr.get_shape()) while not obj_iter.done(obj_state): - reduce_driver.jit_merge_point(shapelen=shapelen, func=func, - done_func=done_func, - calc_dtype=calc_dtype) + reduce_flat_driver.jit_merge_point( + shapelen=shapelen, func=func, + done_func=done_func, calc_dtype=calc_dtype) rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) if done_func is not None and done_func(calc_dtype, rval): return rval @@ -214,33 +255,105 @@ obj_state = obj_iter.next(obj_state) return cur_value -reduce_cum_driver = jit.JitDriver( - name='numpy_reduce_cum_driver', + +reduce_driver = jit.JitDriver( + name='numpy_reduce', + greens=['shapelen', 'func', 'dtype'], reds='auto') + +def reduce(space, func, w_arr, axis_flags, dtype, out, identity): + out_iter, out_state = out.create_iter() + out_iter.track_index = False + shape = w_arr.get_shape() + shapelen = len(shape) + inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags) + assert outer_iter.size == out_iter.size + + if identity is not None: + identity = identity.convert_to(space, dtype) + outer_state = outer_iter.reset() + while not outer_iter.done(outer_state): + inner_state = inner_iter.reset() + inner_state.offset = outer_state.offset + if identity is not None: + w_val = identity + else: + w_val = inner_iter.getitem(inner_state).convert_to(space, dtype) + inner_state = inner_iter.next(inner_state) + while not inner_iter.done(inner_state): + reduce_driver.jit_merge_point( + shapelen=shapelen, func=func, dtype=dtype) + w_item = inner_iter.getitem(inner_state).convert_to(space, dtype) + w_val = func(dtype, w_item, w_val) + inner_state = inner_iter.next(inner_state) + out_iter.setitem(out_state, w_val) + out_state = out_iter.next(out_state) + outer_state = outer_iter.next(outer_state) + return out + +accumulate_flat_driver = jit.JitDriver( + name='numpy_accumulate_flat', greens=['shapelen', 'func', 'dtype', 'out_dtype'], reds='auto') -def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity): - obj_iter, obj_state = obj.create_iter() - out_iter, out_state = out.create_iter() +def accumulate_flat(space, func, w_arr, calc_dtype, w_out, identity): + arr_iter, arr_state = w_arr.create_iter() + out_iter, out_state = w_out.create_iter() out_iter.track_index = False if identity is None: - cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) + cur_value = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) out_iter.setitem(out_state, cur_value) out_state = out_iter.next(out_state) - obj_state = obj_iter.next(obj_state) + arr_state = arr_iter.next(arr_state) else: cur_value = identity.convert_to(space, calc_dtype) - shapelen = len(obj.get_shape()) - out_dtype = out.get_dtype() - while not obj_iter.done(obj_state): - reduce_cum_driver.jit_merge_point( - shapelen=shapelen, func=func, - dtype=calc_dtype, out_dtype=out_dtype) - rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype) - cur_value = func(calc_dtype, cur_value, rval) + shapelen = len(w_arr.get_shape()) + out_dtype = w_out.get_dtype() + while not arr_iter.done(arr_state): + accumulate_flat_driver.jit_merge_point( + shapelen=shapelen, func=func, dtype=calc_dtype, + out_dtype=out_dtype) + w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) + cur_value = func(calc_dtype, cur_value, w_item) out_iter.setitem(out_state, out_dtype.coerce(space, cur_value)) out_state = out_iter.next(out_state) - obj_state = obj_iter.next(obj_state) + arr_state = arr_iter.next(arr_state) + +accumulate_driver = jit.JitDriver( + name='numpy_accumulate', + greens=['shapelen', 'func', 'calc_dtype'], reds='auto') + + +def accumulate(space, func, w_arr, axis, calc_dtype, w_out, identity): + out_iter, out_state = w_out.create_iter() + arr_shape = w_arr.get_shape() + temp_shape = arr_shape[:axis] + arr_shape[axis + 1:] + temp = W_NDimArray.from_shape(space, temp_shape, calc_dtype, w_instance=w_arr) + temp_iter = AxisIter(temp.implementation, w_arr.get_shape(), axis) + temp_state = temp_iter.reset() + arr_iter, arr_state = w_arr.create_iter() + arr_iter.track_index = False + if identity is not None: + identity = identity.convert_to(space, calc_dtype) + shapelen = len(arr_shape) + while not out_iter.done(out_state): + accumulate_driver.jit_merge_point(shapelen=shapelen, func=func, + calc_dtype=calc_dtype) + w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype) + arr_state = arr_iter.next(arr_state) + + out_indices = out_iter.indices(out_state) + if out_indices[axis] == 0: + if identity is not None: + w_item = func(calc_dtype, identity, w_item) + else: + cur_value = temp_iter.getitem(temp_state) + w_item = func(calc_dtype, cur_value, w_item) + + out_iter.setitem(out_state, w_item) + out_state = out_iter.next(out_state) + temp_iter.setitem(temp_state, w_item) + temp_state = temp_iter.next(temp_state) + return w_out def fill(arr, box): arr_iter, arr_state = arr.create_iter() @@ -298,64 +411,56 @@ state = x_state return out -axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', 'func', 'dtype'], - reds='auto') - -def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative, - temp): - out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative) - out_state = out_iter.reset() - if cumulative: - temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False) - temp_state = temp_iter.reset() - else: - temp_iter = out_iter # hack - temp_state = out_state - arr_iter, arr_state = arr.create_iter() - arr_iter.track_index = False - if identity is not None: - identity = identity.convert_to(space, dtype) - shapelen = len(shape) - while not out_iter.done(out_state): - axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func, - dtype=dtype) - w_val = arr_iter.getitem(arr_state).convert_to(space, dtype) - arr_state = arr_iter.next(arr_state) - - out_indices = out_iter.indices(out_state) - if out_indices[axis] == 0: - if identity is not None: - w_val = func(dtype, identity, w_val) - else: - cur = temp_iter.getitem(temp_state) - w_val = func(dtype, cur, w_val) - - out_iter.setitem(out_state, w_val) - out_state = out_iter.next(out_state) - if cumulative: - temp_iter.setitem(temp_state, w_val) - temp_state = temp_iter.next(temp_state) - else: - temp_state = out_state - return out - def _new_argmin_argmax(op_name): arg_driver = jit.JitDriver(name='numpy_' + op_name, greens = ['shapelen', 'dtype'], reds = 'auto') + arg_flat_driver = jit.JitDriver(name='numpy_flat_' + op_name, + greens = ['shapelen', 'dtype'], + reds = 'auto') - def argmin_argmax(arr): + def argmin_argmax(space, w_arr, w_out, axis): + from pypy.module.micronumpy.descriptor import get_dtype_cache + dtype = w_arr.get_dtype() + shapelen = len(w_arr.get_shape()) + axis_flags = [False] * shapelen + axis_flags[axis] = True + inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags) + outer_state = outer_iter.reset() + out_iter, out_state = w_out.create_iter() + while not outer_iter.done(outer_state): + inner_state = inner_iter.reset() + inner_state.offset = outer_state.offset + cur_best = inner_iter.getitem(inner_state) + inner_state = inner_iter.next(inner_state) + result = 0 + idx = 1 + while not inner_iter.done(inner_state): + arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) + w_val = inner_iter.getitem(inner_state) + new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) + if dtype.itemtype.ne(new_best, cur_best): + result = idx + cur_best = new_best + inner_state = inner_iter.next(inner_state) + idx += 1 + result = get_dtype_cache(space).w_longdtype.box(result) + out_iter.setitem(out_state, result) + out_state = out_iter.next(out_state) + outer_state = outer_iter.next(outer_state) + return w_out + + def argmin_argmax_flat(w_arr): result = 0 idx = 1 - dtype = arr.get_dtype() - iter, state = arr.create_iter() + dtype = w_arr.get_dtype() + iter, state = w_arr.create_iter() cur_best = iter.getitem(state) state = iter.next(state) - shapelen = len(arr.get_shape()) + shapelen = len(w_arr.get_shape()) while not iter.done(state): - arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) + arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) w_val = iter.getitem(state) new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) if dtype.itemtype.ne(new_best, cur_best): @@ -364,9 +469,10 @@ state = iter.next(state) idx += 1 return result - return argmin_argmax -argmin = _new_argmin_argmax('min') -argmax = _new_argmin_argmax('max') + + return argmin_argmax, argmin_argmax_flat +argmin, argmin_flat = _new_argmin_argmax('min') +argmax, argmax_flat = _new_argmin_argmax('max') dot_driver = jit.JitDriver(name = 'numpy_dot', greens = ['dtype'], diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -23,6 +23,8 @@ get_shape_from_iterable, shape_agreement, shape_agreement_multiple, is_c_contiguous, is_f_contiguous, calc_strides, new_view) from pypy.module.micronumpy.casting import can_cast_array +from pypy.module.micronumpy.descriptor import get_dtype_cache + def _match_dot_shapes(space, left, right): @@ -484,7 +486,7 @@ return self.implementation.swapaxes(space, self, axis1, axis2) def descr_nonzero(self, space): - index_type = descriptor.get_dtype_cache(space).w_int64dtype + index_type = get_dtype_cache(space).w_int64dtype return self.implementation.nonzero(space, index_type) def descr_tolist(self, space): @@ -544,8 +546,10 @@ def descr_set_flatiter(self, space, w_obj): iter, state = self.create_iter() dtype = self.get_dtype() - arr = convert_to_array(space, w_obj) - loop.flatiter_setitem(space, dtype, arr, iter, state, 1, iter.size) + w_arr = convert_to_array(space, w_obj) + if dtype.is_record(): + return self.implementation.setslice(space, w_arr) + loop.flatiter_setitem(space, dtype, w_arr, iter, state, 1, iter.size) def descr_get_flatiter(self, space): from .flatiter import W_FlatIterator @@ -810,7 +814,7 @@ if self.get_dtype().is_bool(): # numpy promotes bool.round() to float16. Go figure. w_out = W_NDimArray.from_shape(space, self.get_shape(), - descriptor.get_dtype_cache(space).w_float16dtype) + get_dtype_cache(space).w_float16dtype) else: w_out = None elif not isinstance(w_out, W_NDimArray): @@ -818,7 +822,7 @@ "return arrays must be of ArrayType")) out = descriptor.dtype_agreement(space, [self], self.get_shape(), w_out) if out.get_dtype().is_bool() and self.get_dtype().is_bool(): - calc_dtype = descriptor.get_dtype_cache(space).w_longdtype + calc_dtype = get_dtype_cache(space).w_longdtype else: calc_dtype = out.get_dtype() @@ -837,7 +841,7 @@ raise oefmt(space.w_ValueError, "a must be a 1-d array") v = convert_to_array(space, w_v) ret = W_NDimArray.from_shape( - space, v.get_shape(), descriptor.get_dtype_cache(space).w_longdtype) + space, v.get_shape(), get_dtype_cache(space).w_longdtype) if side == NPY.SEARCHLEFT: binsearch = loop.binsearch_left else: @@ -1145,35 +1149,46 @@ # ----------------------- reduce ------------------------------- - def _reduce_ufunc_impl(ufunc_name, cumulative=False, bool_result=False): + def _reduce_ufunc_impl(ufunc_name, name, bool_result=False): @unwrap_spec(keepdims=bool) def impl(self, space, w_axis=None, w_dtype=None, w_out=None, keepdims=False): out = out_converter(space, w_out) if bool_result: - w_dtype = descriptor.get_dtype_cache(space).w_booldtype + w_dtype = get_dtype_cache(space).w_booldtype return getattr(ufuncs.get(space), ufunc_name).reduce( From noreply at buildbot.pypy.org Tue Aug 11 16:17:43 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 16:17:43 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): also support value profiling of instance attributes Message-ID: <20150811141743.4C9E21C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78897:5a064b6c607a Date: 2015-08-11 16:08 +0200 http://bitbucket.org/pypy/pypy/changeset/5a064b6c607a/ Log: (arigo, cfbolz): also support value profiling of instance attributes diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -47,8 +47,12 @@ if status != SEEN_TOO_MUCH: self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_NOTHING: - self._vprof_value_wref = ref(value) - self._vprof_status = SEEN_OBJ + try: + self._vprof_value_wref = ref(value) + self._vprof_status = SEEN_OBJ + except TypeError: + # for tests, which really use unwrapped ints in a few places + self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_INT: self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_OBJ: diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -4,6 +4,7 @@ from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter import valueprof from pypy.objspace.std.dictmultiobject import ( W_DictMultiObject, DictStrategy, ObjectDictStrategy, BaseKeyIterator, BaseValueIterator, BaseItemIterator, _never_equal_to_string @@ -30,9 +31,17 @@ self.terminator = terminator def read(self, obj, selector): + from pypy.objspace.std.intobject import W_IntObject attr = self.find_map_attr(selector) if attr is None: return self.terminator._read_terminator(obj, selector) + # XXX move to PlainAttribute? + if attr.can_fold_read_int(): + return W_IntObject(attr.read_constant_int()) + elif attr.can_fold_read_obj(): + w_res = attr.try_read_constant_obj() + if w_res is not None: + return w_res if ( jit.isconstant(attr.storageindex) and jit.isconstant(obj) and @@ -50,6 +59,7 @@ attr = self.find_map_attr(selector) if attr is None: return self.terminator._write_terminator(obj, selector, w_value) + attr.see_write(w_value) if not attr.ever_mutated: attr.ever_mutated = True obj._mapdict_write_storage(attr.storageindex, w_value) @@ -170,6 +180,7 @@ # for the benefit of the special subclasses obj._set_mapdict_map(attr) obj._mapdict_write_storage(attr.storageindex, w_value) + attr.see_write(w_value) def materialize_r_dict(self, space, obj, dict_w): raise NotImplementedError("abstract base class") @@ -274,8 +285,10 @@ terminator = terminator.devolved_dict_terminator return Terminator.set_terminator(self, obj, terminator) + class PlainAttribute(AbstractAttribute): _immutable_fields_ = ['selector', 'storageindex', 'back', 'ever_mutated?'] + objectmodel.import_from_mixin(valueprof.ValueProf) def __init__(self, selector, back): AbstractAttribute.__init__(self, back.space, back.terminator) @@ -284,6 +297,19 @@ self.back = back self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False + self.init_valueprof() + + # ____________________________________________________________ + # methods for ValueProf mixin + def is_int(self, w_obj): + from pypy.objspace.std.intobject import W_IntObject + return type(w_obj) is W_IntObject + + def get_int_val(self, w_obj): + from pypy.objspace.std.intobject import W_IntObject + assert isinstance(w_obj, W_IntObject) + return w_obj.intval + # ____________________________________________________________ def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.selector) diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -348,6 +348,31 @@ obj.setdictvalue(space, a, 50) assert c.terminator.size_estimate() in [(i + 10) // 2, (i + 11) // 2] +def test_value_profiling(monkeypatch): + class Value: + pass + a = Value() + cls = Class() + obj = cls.instantiate() + obj.setdictvalue(space, "a", a) + obj = cls.instantiate() + obj.setdictvalue(space, "a", a) + obj.setdictvalue(space, "a", a) + + def _mapdict_read_storage(storageindex): + assert 0 # not reached + + obj._mapdict_read_storage = _mapdict_read_storage + + assert obj.getdictvalue(space, "a") == a + assert obj.getdictvalue(space, "a") == a + + obj = cls.instantiate() + obj.setdictvalue(space, "a", a) + obj.setdictvalue(space, "a", Value()) + assert not obj.map.can_fold_read_obj() + + # ___________________________________________________________ # dict tests From noreply at buildbot.pypy.org Tue Aug 11 16:17:45 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 16:17:45 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): temporarily add some debug printing Message-ID: <20150811141745.5EC321C089E@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78898:77cb21016bed Date: 2015-08-11 16:17 +0200 http://bitbucket.org/pypy/pypy/changeset/77cb21016bed/ Log: (arigo, cfbolz): temporarily add some debug printing diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -95,7 +95,7 @@ self._signature = cpython_code_signature(self) self._initialize() self._init_ready() - self.vprofs = [ValueProf() for i in range(self.co_nlocals)] + self.vprofs = [ValueProf('%s %s' % (self.co_name, self.co_varnames[i])) for i in range(self.co_nlocals)] def _initialize(self): if self.co_cellvars: diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -10,14 +10,16 @@ _mixin_ = True _immutable_fields_ = ['_vprof_status?'] - def __init__(self): + def __init__(self, msg=''): # only if you subclass normally - self.init_valueprof() + self.init_valueprof(msg) - def init_valueprof(self): + def init_valueprof(self, msg=''): self._vprof_status = SEEN_NOTHING self._vprof_value_int = 0 self._vprof_value_wref = dead_ref + self._vprof_counter = 0 + self._vprof_msg = msg def is_int(self, w_obj): raise NotImplementedError("abstract base") @@ -38,6 +40,10 @@ elif status == SEEN_INT: if self.read_constant_int() != value: self._vprof_status = SEEN_TOO_MUCH + else: + self._vprof_counter += 1 + if self._vprof_counter == 200: + print self._vprof_msg, 'int', value elif status == SEEN_OBJ: self._vprof_status = SEEN_TOO_MUCH @@ -58,6 +64,10 @@ elif status == SEEN_OBJ: if self.try_read_constant_obj() is not value: self._vprof_status = SEEN_TOO_MUCH + else: + self._vprof_counter += 1 + if self._vprof_counter == 200: + print self._vprof_msg, 'obj', value def can_fold_read_int(self): return self._vprof_status == SEEN_INT diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -297,7 +297,7 @@ self.back = back self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False - self.init_valueprof() + self.init_valueprof('%s.%s' % (back.terminator.w_cls.name, selector[0])) # ____________________________________________________________ # methods for ValueProf mixin From noreply at buildbot.pypy.org Tue Aug 11 16:39:00 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 16:39:00 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: added branch description for vecopt and vecopt-merge Message-ID: <20150811143900.1391A1C12AA@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78899:98e339e44deb Date: 2015-08-11 16:39 +0200 http://bitbucket.org/pypy/pypy/changeset/98e339e44deb/ Log: added branch description for vecopt and vecopt-merge diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -63,3 +63,8 @@ .. branch: fix-tinylang-goals Update tinylang goals to match current rpython + +.. branch: vecopt +.. branch: vecopt-merge + +A new optimization pass to use SIMD instructions for trace loop that allow this From noreply at buildbot.pypy.org Tue Aug 11 16:41:10 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 16:41:10 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (cfbolz, arigo): grumble grumble Message-ID: <20150811144110.6BC731C0206@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78900:3b5a634dfc67 Date: 2015-08-11 16:40 +0200 http://bitbucket.org/pypy/pypy/changeset/3b5a634dfc67/ Log: (cfbolz, arigo): grumble grumble diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -297,7 +297,7 @@ self.back = back self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False - self.init_valueprof('%s.%s' % (back.terminator.w_cls.name, selector[0])) + self.init_valueprof('%s.%s' % (back.terminator.w_cls.name if back.terminator.w_cls else '???', selector[0])) # ____________________________________________________________ # methods for ValueProf mixin From noreply at buildbot.pypy.org Tue Aug 11 16:46:09 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 16:46:09 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): this is not needed at all any more Message-ID: <20150811144609.9A4BC1C0206@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78901:54c6274f6261 Date: 2015-08-11 16:46 +0200 http://bitbucket.org/pypy/pypy/changeset/54c6274f6261/ Log: (arigo, cfbolz): this is not needed at all any more diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -7,7 +7,6 @@ SEEN_TOO_MUCH = '\x03' class ValueProf(object): - _mixin_ = True _immutable_fields_ = ['_vprof_status?'] def __init__(self, msg=''): From noreply at buildbot.pypy.org Tue Aug 11 16:51:07 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 16:51:07 +0200 (CEST) Subject: [pypy-commit] pypy default: (cfbolz around, arigo) Detect when we use at the same time the old and the new style of mixins Message-ID: <20150811145107.AE2C01C0206@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r78902:deb3c9fb551c Date: 2015-08-11 15:51 +0100 http://bitbucket.org/pypy/pypy/changeset/deb3c9fb551c/ Log: (cfbolz around, arigo) Detect when we use at the same time the old and the new style of mixins diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -839,6 +839,9 @@ if key in target: raise Exception("import_from_mixin: would overwrite the value " "already defined locally for %r" % (key,)) + if key == '_mixin_': + raise Exception("import_from_mixin(M): class M should not " + "have '_mixin_ = True'") target[key] = value if immutable_fields: target['_immutable_fields_'] = target.get('_immutable_fields_', []) + immutable_fields From noreply at buildbot.pypy.org Tue Aug 11 16:56:33 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:33 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: implement a proper fix for heapcache dealing with getfield_gc_pure Message-ID: <20150811145633.DE6761C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78903:18a9d18296d2 Date: 2015-08-07 14:36 +0200 http://bitbucket.org/pypy/pypy/changeset/18a9d18296d2/ Log: implement a proper fix for heapcache dealing with getfield_gc_pure diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -116,6 +116,10 @@ self.fieldname = fieldname self.FIELD = getattr(S, fieldname) self.index = heaptracker.get_fielddescr_index_in(S, fieldname) + self._is_pure = S._immutable_field(fieldname) + + def is_always_pure(self): + return self._is_pure def get_parent_descr(self): return self.parent_descr diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -287,11 +287,13 @@ def clean_caches(self): del self._lazy_setfields_and_arrayitems[:] for descr, cf in self.cached_fields.items(): - cf.invalidate(descr) - for submap in self.cached_arrayitems.itervalues(): - for index, cf in submap.iteritems(): - cf.invalidate(None) - self.cached_arrayitems.clear() + if not descr.is_always_pure(): + cf.invalidate(descr) + for descr, submap in self.cached_arrayitems.iteritems(): + if not descr.is_always_pure(): + for index, cf in submap.iteritems(): + cf.invalidate(None) + #self.cached_arrayitems.clear() self.cached_dict_reads.clear() def field_cache(self, descr): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -268,6 +268,9 @@ def force_op_from_preamble(self, op): return op + def notice_guard_future_condition(self, op): + self.patchguardop = op + def replace_guard(self, op, value): assert isinstance(value, info.NonNullPtrInfo) if value.last_guard_pos == -1: diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -593,7 +593,7 @@ self.emit_operation(op) def optimize_GUARD_FUTURE_CONDITION(self, op): - pass # just remove it + self.optimizer.notice_guard_future_condition(op) def optimize_INT_FLOORDIV(self, op): arg0 = op.getarg(0) diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py --- a/rpython/jit/metainterp/optimizeopt/simplify.py +++ b/rpython/jit/metainterp/optimizeopt/simplify.py @@ -70,7 +70,7 @@ # self.emit_operation(op) def optimize_GUARD_FUTURE_CONDITION(self, op): - pass + self.optimizer.notice_guard_future_condition(op) dispatch_opt = make_dispatcher_method(OptSimplify, 'optimize_', default=OptSimplify.emit_operation) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2949,7 +2949,7 @@ """ exc = self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") if exc: - assert "node" in exc.msg + assert "promote of a virtual" in exc.msg def test_merge_guard_class_guard_value(self): ops = """ @@ -3171,8 +3171,6 @@ [p1, p2] i1 = ptr_eq(p1, p2) i3 = int_add(i1, 1) - i3b = int_is_true(i3) - guard_true(i3b) [] escape_n(i3) escape_n(i3) guard_true(i1) [] @@ -7438,17 +7436,17 @@ ops = """ [p0] - p1 = new_with_vtable(descr=nodesize) - setfield_gc(p1, p0, descr=valuedescr) + p1 = new_with_vtable(descr=nodesize3) + setfield_gc(p1, p0, descr=valuedescr3) escape_n(p1) - p2 = getfield_gc_pure_r(p1, descr=valuedescr) + p2 = getfield_gc_pure_r(p1, descr=valuedescr3) escape_n(p2) jump(p0) """ expected = """ [p0] - p1 = new_with_vtable(descr=nodesize) - setfield_gc(p1, p0, descr=valuedescr) + p1 = new_with_vtable(descr=nodesize3) + setfield_gc(p1, p0, descr=valuedescr3) escape_n(p1) escape_n(p0) jump(p0) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -88,6 +88,9 @@ node_vtable2 = lltype.malloc(OBJECT_VTABLE, immortal=True) node_vtable2.name = rclass.alloc_array_name('node2') node_vtable_adr2 = llmemory.cast_ptr_to_adr(node_vtable2) + node_vtable3 = lltype.malloc(OBJECT_VTABLE, immortal=True) + node_vtable3.name = rclass.alloc_array_name('node3') + node_vtable_adr3 = llmemory.cast_ptr_to_adr(node_vtable3) cpu = runner.LLGraphCPU(None) NODE = lltype.GcForwardReference() @@ -98,6 +101,13 @@ ('next', lltype.Ptr(NODE)))) NODE2 = lltype.GcStruct('NODE2', ('parent', NODE), ('other', lltype.Ptr(NODE))) + + NODE3 = lltype.GcForwardReference() + NODE3.become(lltype.GcStruct('NODE3', ('parent', OBJECT), + ('value', lltype.Signed), + ('next', lltype.Ptr(NODE3)), + hints={'immutable': True})) + node = lltype.malloc(NODE) node.value = 5 node.parent.typeptr = node_vtable @@ -111,11 +121,16 @@ #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2)) nodesize = cpu.sizeof(NODE, True) nodesize2 = cpu.sizeof(NODE2, True) + nodesize3 = cpu.sizeof(NODE3, True) valuedescr = cpu.fielddescrof(NODE, 'value') floatdescr = cpu.fielddescrof(NODE, 'floatval') chardescr = cpu.fielddescrof(NODE, 'charval') nextdescr = cpu.fielddescrof(NODE, 'next') otherdescr = cpu.fielddescrof(NODE2, 'other') + valuedescr3 = cpu.fielddescrof(NODE3, 'value') + nextdescr3 = cpu.fielddescrof(NODE3, 'next') + assert valuedescr3.is_always_pure() + assert nextdescr3.is_always_pure() accessor = FieldListAccessor() accessor.initialize(None, {'inst_field': IR_QUASIIMMUTABLE}) @@ -312,6 +327,7 @@ register_known_gctype(cpu, node_vtable, NODE) register_known_gctype(cpu, node_vtable2, NODE2) + register_known_gctype(cpu, node_vtable3, NODE3) register_known_gctype(cpu, u_vtable, U) register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF) register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT) @@ -433,6 +449,7 @@ return call_pure_results def unroll_and_optimize(self, loop, call_pure_results=None): + self.add_guard_future_condition(loop) jump_op = loop.operations[-1] assert jump_op.getopnum() == rop.JUMP ops = loop.operations[:-1] diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -125,7 +125,12 @@ for i in range(len(jump_args)): sb.short_inputargs[i].set_forwarded(None) self.make_equal_to(sb.short_inputargs[i], jump_args[i]) + patchguardop = self.optimizer.patchguardop for op in sb.short: + if op.is_guard(): + op = self.replace_op_with(op, op.getopnum()) + op.rd_snapshot = patchguardop.rd_snapshot + op.rd_frame_info_list = patchguardop.rd_frame_info_list self.optimizer.send_extra_operation(op) res = [self.optimizer.get_box_replacement(op) for op in sb.short_preamble_jump] diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -170,17 +170,6 @@ def optimize_GETFIELD_GC_I(self, op): opinfo = self.getptrinfo(op.getarg(0)) - # XXX dealt with by heapcache - # If this is an immutable field (as indicated by op.is_always_pure()) - # then it's safe to reuse the virtual's field, even if it has been - # forced, because it should never be written to again. - #if op.is_always_pure(): - # - # if value.is_forced_virtual() and op.is_always_pure(): - # fieldvalue = value.getfield(op.getdescr(), None) - # if fieldvalue is not None: - # self.make_equal_to(op, fieldvalue) - # return if opinfo and opinfo.is_virtual(): fieldop = opinfo.getfield(op.getdescr()) if fieldop is None: From noreply at buildbot.pypy.org Tue Aug 11 16:56:36 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:36 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix test Message-ID: <20150811145636.1B7501C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78904:67c0fb707827 Date: 2015-08-07 14:39 +0200 http://bitbucket.org/pypy/pypy/changeset/67c0fb707827/ Log: fix test diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -172,9 +172,13 @@ def __init__(self, A, runner): self.A = self.OUTERA = A + self._is_pure = A._immutable_field(None) if isinstance(A, lltype.Struct): self.A = A._flds[A._arrayfld] + def is_always_pure(self): + return self._is_pure + def get_all_fielddescrs(self): return self.all_interiorfielddescrs diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7369,7 +7369,7 @@ ops = """ [p0, p1, p3, ii, ii2] i1 = getfield_gc_i(p0, descr=valuedescr) - i2 = getfield_gc_i(p1, descr=otherdescr) + i2 = getfield_gc_i(p1, descr=chardescr) setfield_gc(p3, i1, descr=adescr) setfield_gc(p3, i2, descr=bdescr) i4 = int_eq(i1, i2) From noreply at buildbot.pypy.org Tue Aug 11 16:56:38 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:38 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix tests Message-ID: <20150811145638.361651C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78905:456183623e74 Date: 2015-08-09 11:18 +0200 http://bitbucket.org/pypy/pypy/changeset/456183623e74/ Log: fix tests diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -198,9 +198,9 @@ def _getfield(self, opinfo, descr, optheap): res = opinfo.getitem(self.index, optheap) if isinstance(res, PreambleOp): + index = res.preamble_op.getarg(1).getint() res = optheap.optimizer.force_op_from_preamble(res) - opinfo.setitem(res.getarg(1).getint(), None, res, - optheap=optheap) + opinfo.setitem(index, None, res, optheap=optheap) return res def _setfield(self, op, opinfo, optheap): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7413,8 +7413,8 @@ jump(p0) """ expected = """ - [p0] - jump(p0) + [p0, p1] + jump(p0, p1) """ short = """ [p0] From noreply at buildbot.pypy.org Tue Aug 11 16:56:40 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:40 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix a few tests and improve one Message-ID: <20150811145640.4E6181C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78906:13aafd39caf2 Date: 2015-08-09 11:41 +0200 http://bitbucket.org/pypy/pypy/changeset/13aafd39caf2/ Log: fix a few tests and improve one diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -147,10 +147,11 @@ for short_op in self.const_short_boxes: getfield_op = short_op.getfield_op - preamble_arg = self.produce_arg(getfield_op.getarg(0)) + args = getfield_op.getarglist() + preamble_arg = self.produce_arg(args[0]) if preamble_arg is not None: preamble_op = getfield_op.copy_and_change( - getfield_op.getopnum(), [preamble_arg]) + getfield_op.getopnum(), [preamble_arg] + args[1:]) produced_op = ProducedShortOp(short_op, preamble_op) short_boxes.append(produced_op) return short_boxes diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3997,7 +3997,7 @@ [p0, i1] p3 = force_token() # - p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + p2 = new_with_vtable(descr=vref_descr) setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) @@ -4085,7 +4085,7 @@ expected = """ [i1] p3 = force_token() - p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + p2 = new_with_vtable(descr=vref_descr) setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) escape_n(p2) @@ -7223,36 +7223,38 @@ def test_constant_getfield1(self): ops = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=) + p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) + call_n(123, p25, descr=nonwritedescr) jump(p25, p187, i184) """ preamble = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=) + p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) - p26 = same_as_r(p25) - jump(p25, p187, i184, p26) + p123 = same_as_r(p25) + jump(p25, p187, i184, p123) """ short = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=) + p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) - jump(p1, p187, i184, p25) + jump(p25) """ expected = """ [p25, p187, i184, p189] - jump(p189, p187, i184, p189) + call_n(123, p189, descr=nonwritedescr) + jump(p25, p187, i184, p189) """ self.optimize_loop(ops, expected, preamble, expected_short=short) def test_constant_getfield1bis(self): ops = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=) + p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) p26 = call_r(p25, descr=nonwritedescr) From noreply at buildbot.pypy.org Tue Aug 11 16:56:42 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:42 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix some tests Message-ID: <20150811145642.623D01C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78907:ca67a935c149 Date: 2015-08-11 12:34 +0200 http://bitbucket.org/pypy/pypy/changeset/ca67a935c149/ Log: fix some tests diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -52,8 +52,7 @@ return None def make_guards(self, op, short): - pass # XXX - + pass class NonNullPtrInfo(PtrInfo): _attrs_ = ('last_guard_pos',) @@ -85,6 +84,10 @@ return return self._visitor_walk_recursive(instbox, visitor, optimizer) + def make_guards(self, op, short): + op = ResOperation(rop.GUARD_NONNULL, [op], None) + short.append(op) + class AbstractVirtualPtrInfo(NonNullPtrInfo): _attrs_ = ('_cached_vinfo', 'vdescr') # XXX merge _cached_vinfo with vdescr diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1278,9 +1278,9 @@ [i0, p1, p3] i28 = int_add(i0, 1) p30 = new_with_vtable(descr=nodesize) - setfield_gc(p30, i28, descr=nextdescr) - setfield_gc(p3, p30, descr=valuedescr) - p45 = getfield_gc_r(p3, descr=valuedescr) + setfield_gc(p30, i28, descr=valuedescr) + setfield_gc(p3, p30, descr=nextdescr) + p45 = getfield_gc_r(p3, descr=nextdescr) i29 = int_add(i28, 1) jump(i29, p45, p3) """ @@ -1289,8 +1289,8 @@ i28 = int_add(i0, 1) i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) - setfield_gc(p30, i28, descr=nextdescr) - setfield_gc(p3, p30, descr=valuedescr) + setfield_gc(p30, i28, descr=valuedescr) + setfield_gc(p3, p30, descr=nextdescr) #p46 = same_as(p30) # This same_as should be killed by backend jump(i29, p30, p3) """ @@ -1299,8 +1299,8 @@ i28 = int_add(i0, 1) i29 = int_add(i0, 2) p30 = new_with_vtable(descr=nodesize) - setfield_gc(p30, i28, descr=nextdescr) - setfield_gc(p3, p30, descr=valuedescr) + setfield_gc(p30, i28, descr=valuedescr) + setfield_gc(p3, p30, descr=nextdescr) jump(i29, p30, p3) """ self.optimize_loop(ops, expected, preamble) @@ -3924,12 +3924,12 @@ expected = """ [p1] p0 = force_token() - p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + p2 = new_with_vtable(descr=vref_descr) + setfield_gc(p2, p0, descr=virtualtokendescr) setfield_gc(p2, NULL, descr=virtualforceddescr) - setfield_gc(p2, p0, descr=virtualtokendescr) escape_n(p2) + setfield_gc(p2, NULL, descr=virtualtokendescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p1) """ # XXX we should optimize a bit more the case of a nonvirtual. @@ -4273,13 +4273,17 @@ i2 = int_lt(i0, 5) jump(i2) """ - expected = """ + preamble = """ [i0] i1 = int_lt(i0, 4) i2 = int_lt(i0, 5) jump(i2) """ - self.optimize_loop(ops, expected, expected) + expected = """ + [i0] + jump(1) + """ + self.optimize_loop(ops, expected, preamble) def test_bound_lt_noopt(self): ops = """ From noreply at buildbot.pypy.org Tue Aug 11 16:56:44 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:44 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix one more test Message-ID: <20150811145644.793361C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78908:8505df96400f Date: 2015-08-11 13:11 +0200 http://bitbucket.org/pypy/pypy/changeset/8505df96400f/ Log: fix one more test diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -562,10 +562,9 @@ indexb = self.getintbound(op.getarg(1)) cf = None if indexb.is_constant(): - # XXXX lgt bound - #arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint()) + index = indexb.getint() + arrayinfo.getlenbound().make_gt_const(index) # use the cache on (arraydescr, index), which is a constant - index = indexb.getint() cf = self.arrayitem_cache(op.getdescr(), index) field = cf.getfield_from_cache(self, arrayinfo, op.getdescr()) if field is not None: @@ -592,11 +591,11 @@ indexb = self.getintbound(op.getarg(1)) cf = None if indexb.is_constant(): - #arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint()) + index = indexb.getint() + arrayinfo.getlenbound().make_gt_const(index) # use the cache on (arraydescr, index), which is a constant - #cf = self.arrayitem_cache(op.getdescr(), indexvalue.box.getint()) - #fieldvalue = cf.getfield_from_cache(self, arrayvalue) - fieldvalue = None + cf = self.arrayitem_cache(op.getdescr(), index) + fieldvalue = cf.getfield_from_cache(self, arrayinfo, op.getdescr()) if fieldvalue is not None: self.make_equal_to(op, fieldvalue) return @@ -620,10 +619,9 @@ # indexb = self.getintbound(op.getarg(1)) if indexb.is_constant(): - #arrayinfo = self.ensure_ptr_info_arg0(op) + arrayinfo = self.ensure_ptr_info_arg0(op) # arraybound - #arrayvalue.make_len_gt(MODE_ARRAY, op.getdescr(), indexvalue.box.getint()) - # use the cache on (arraydescr, index), which is a constant + arrayinfo.getlenbound().make_gt_const(indexb.getint()) cf = self.arrayitem_cache(op.getdescr(), indexb.getint()) cf.do_setfield(self, op) else: diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -301,16 +301,20 @@ length = -1 def __init__(self, const=None, size=0, clear=False, vdescr=None): + from rpython.jit.metainterp.optimizeopt import intutils self.vdescr = vdescr if vdescr is not None: self._init_items(const, size, clear) + self.lenbound = intutils.ConstIntBound(size) self._clear = clear - #def getlenbound(self): - # if self.lenbound is None: - # raise Exception("implement me - lenbound") - # xxx - # return self.lenbound + def getlenbound(self): + from rpython.jit.metainterp.optimizeopt import intutils + + if self.lenbound is None: + assert self.length == -1 + self.lenbound = intutils.IntLowerBound(0) + return self.lenbound def _init_items(self, const, size, clear): self.length = size @@ -378,6 +382,13 @@ descr=descr) shortboxes.add_heap_op(op, getarrayitem_op) + def make_guards(self, op, short): + AbstractVirtualPtrInfo.make_guards(self, op, short) + if self.lenbound is not None: + lenop = ResOperation(rop.ARRAYLEN_GC, [op]) + short.append(lenop) + self.lenbound.make_guards(lenop, short) + class ArrayStructInfo(ArrayPtrInfo): def __init__(self, size, vdescr=None): self.length = size diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py --- a/rpython/jit/metainterp/optimizeopt/intbounds.py +++ b/rpython/jit/metainterp/optimizeopt/intbounds.py @@ -409,12 +409,9 @@ bres.intersect(bounds) def optimize_ARRAYLEN_GC(self, op): - array = self.getptrinfo(op.getarg(0)) - result = self.getintbound(op) - result.make_ge(IntLowerBound(0)) + array = self.ensure_ptr_info_arg0(op) self.emit_operation(op) - #array.make_len_gt(MODE_ARRAY, op.getdescr(), -1) - #array.getlenbound().bound.intersect(result.getintbound()) + self.optimizer.setintbound(op, array.getlenbound()) def optimize_STRLEN(self, op): self.emit_operation(op) diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py --- a/rpython/jit/metainterp/optimizeopt/intutils.py +++ b/rpython/jit/metainterp/optimizeopt/intutils.py @@ -43,6 +43,18 @@ return True return False + def make_ge_const(self, other): + return self.make_ge(ConstIntBound(other)) + + def make_gt_const(self, other): + return self.make_gt(ConstIntBound(other)) + + def make_eq_const(self, intval): + self.has_upper = True + self.has_lower = True + self.upper = intval + self.lower = intval + def make_gt(self, other): return self.make_ge(other.add(1)) diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -59,6 +59,14 @@ op.set_forwarded(intbound) return intbound + def setintbound(self, op, bound): + assert op.type == 'i' + op = self.get_box_replacement(op) + if op.is_constant(): + return + assert op.get_forwarded() is None + op.set_forwarded(bound) + def getnullness(self, op): if op.type == 'i': return self.getintbound(op).getnullness() @@ -425,7 +433,8 @@ opinfo = info.StructPtrInfo() opinfo.init_fields(op.getdescr().get_parent_descr(), op.getdescr().get_index()) - elif op.is_getarrayitem() or op.getopnum() == rop.SETARRAYITEM_GC: + elif (op.is_getarrayitem() or op.getopnum() == rop.SETARRAYITEM_GC or + op.getopnum() == rop.ARRAYLEN_GC): opinfo = info.ArrayPtrInfo(op.getdescr()) elif op.getopnum() in (rop.GUARD_CLASS, rop.GUARD_NONNULL_CLASS): opinfo = info.InstancePtrInfo() diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4659,19 +4659,41 @@ def test_bound_arraylen(self): ops = """ [i0, p0] - p1 = new_array(i0, descr=arraydescr) + p1 = new_array(i0, descr=gcarraydescr) i1 = arraylen_gc(p1) i2 = int_gt(i1, -1) guard_true(i2) [] - setarrayitem_gc(p0, 0, p1) + setarrayitem_gc(p0, 0, p1, descr=gcarraydescr) jump(i0, p0) """ # The dead arraylen_gc will be eliminated by the backend. expected = """ [i0, p0] - p1 = new_array(i0, descr=arraydescr) + p1 = new_array(i0, descr=gcarraydescr) i1 = arraylen_gc(p1) - setarrayitem_gc(p0, 0, p1) + setarrayitem_gc(p0, 0, p1, descr=gcarraydescr) + jump(i0, p0) + """ + self.optimize_loop(ops, expected) + + def test_bound_arraylen_2(self): + ops = """ + [i0, p0] + p1 = new_array(i0, descr=gcarraydescr) + setarrayitem_gc(p1, 3, p1, descr=gcarraydescr) + i1 = arraylen_gc(p1) + i2 = int_gt(i1, 2) + guard_true(i2) [] + setarrayitem_gc(p1, 0, p0, descr=gcarraydescr) + jump(i0, p0) + """ + # The dead arraylen_gc will be eliminated by the backend. + expected = """ + [i0, p0] + p1 = new_array(i0, descr=gcarraydescr) + i1 = arraylen_gc(p1) + setarrayitem_gc(p1, 0, p0, descr=gcarraydescr) + setarrayitem_gc(p1, 3, p1, descr=gcarraydescr) jump(i0, p0) """ self.optimize_loop(ops, expected) @@ -7612,7 +7634,7 @@ ops = """ [p0] p1 = getfield_gc_r(p0, descr=nextdescr) - p2 = getarrayitem_gc_r(p1, 7, descr=arraydescr) + p2 = getarrayitem_gc_r(p1, 7, descr=gcarraydescr) call_n(p2, descr=nonwritedescr) jump(p0) """ @@ -7623,13 +7645,13 @@ i1 = arraylen_gc(p1) i2 = int_ge(i1, 8) guard_true(i2) [] - p2 = getarrayitem_gc_r(p1, 7, descr=arraydescr) - jump(p0, p1, p2) + p2 = getarrayitem_gc_r(p1, 7, descr=gcarraydescr) + jump(p1, p2) """ expected = """ [p0, p1, p2] call_n(p2, descr=nonwritedescr) - # i3 = arraylen_gc(p1) # Should be killed by backend + i3 = arraylen_gc(p1) # Should be killed by backend jump(p0, p1, p2) """ self.optimize_loop(ops, expected, expected_short=short) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -168,6 +168,7 @@ immut_ptrval = cpu.fielddescrof(PTROBJ_IMMUT, 'ptrval') arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed)) + gcarraydescr = cpu.arraydescrof(lltype.GcArray(llmemory.GCREF)) floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float)) # a GcStruct not inheriting from OBJECT From noreply at buildbot.pypy.org Tue Aug 11 16:56:46 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:46 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix another test Message-ID: <20150811145646.840EF1C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78909:df8aedaea834 Date: 2015-08-11 13:21 +0200 http://bitbucket.org/pypy/pypy/changeset/df8aedaea834/ Log: fix another test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7763,14 +7763,14 @@ i2 = int_ge(i1, 8) guard_true(i2) [] p2 = getarrayitem_gc_pure_r(p1, 7, descr=) + jump(p2, p1) + """ + expected = """ + [p0, p2, p1] + call_n(p2, descr=nonwritedescr) + i3 = arraylen_gc(p1) # Should be killed by backend jump(p0, p2, p1) """ - expected = """ - [p0, p2] - call_n(p2, descr=nonwritedescr) - #i3 = arraylen_gc(p1) # Should be killed by backend - jump(p0, p2) - """ self.optimize_loop(ops, expected, expected_short=short) def test_loopinvariant_constant_strgetitem(self): diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -38,6 +38,10 @@ known_class = preamble_info.get_known_class(self.cpu) if known_class: self.make_constant_class(op, known_class, False) + if isinstance(preamble_info, info.ArrayPtrInfo): + arr_info = info.ArrayPtrInfo(None) + arr_info.lenbound = preamble_info.getlenbound() + op.set_forwarded(arr_info) if preamble_info.is_nonnull(): self.make_nonnull(op) elif isinstance(preamble_info, intutils.IntBound): From noreply at buildbot.pypy.org Tue Aug 11 16:56:48 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:48 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix test_bug_3 Message-ID: <20150811145648.9B0101C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78910:7c3ad0460623 Date: 2015-08-11 14:53 +0200 http://bitbucket.org/pypy/pypy/changeset/7c3ad0460623/ Log: fix test_bug_3 diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -128,6 +128,9 @@ assert not self.is_virtual() self._fields = [None] * len(self._fields) + def all_items(self): + return self._fields + def setfield(self, descr, struct, op, optheap=None, cf=None): self.init_fields(descr.get_parent_descr(), descr.get_index()) assert isinstance(op, AbstractValue) @@ -175,7 +178,6 @@ getfield_op = ResOperation(opnum, [structbox], descr=descr) shortboxes.add_heap_op(op, getfield_op) - class InstancePtrInfo(AbstractStructPtrInfo): _attrs_ = ('_known_class',) _fields = None @@ -193,6 +195,14 @@ assert self.is_virtual() return visitor.visit_virtual(self.vdescr, fielddescrs) + def make_guards(self, op, short): + if self._known_class is not None: + op = ResOperation(rop.GUARD_NONNULL_CLASS, [op, self._known_class], + None) + short.append(op) + else: + AbstractStructPtrInfo.make_guards(self, op, short) + class StructPtrInfo(AbstractStructPtrInfo): def __init__(self, vdescr=None): self.vdescr = vdescr diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2776,7 +2776,7 @@ p3 = getfield_gc_r(p1, descr=otherdescr) guard_nonnull(ConstPtr(nodeaddr)) [] guard_class(p3, ConstClass(node_vtable)) [] - setfield_gc(p3, p2, descr=otherdescr) + setfield_gc(p3, p2, descr=nextdescr) p1a = new_with_vtable(descr=nodesize2) p2a = new_with_vtable(descr=nodesize) p3a = new_with_vtable(descr=nodesize) @@ -2793,7 +2793,7 @@ p3 = getfield_gc_r(p1, descr=otherdescr) guard_class(p3, ConstClass(node_vtable)) [] p3a = new_with_vtable(descr=nodesize) - setfield_gc(p3, p2, descr=otherdescr) + setfield_gc(p3, p2, descr=nextdescr) escape_n(p3a) jump(p3a) """ @@ -2807,7 +2807,7 @@ # p2a = new_with_vtable(descr=nodesize) p3anew = new_with_vtable(descr=nodesize) p2 = new_with_vtable(descr=nodesize) - setfield_gc(p3a, p2, descr=otherdescr) # p3a.other = p2a + setfield_gc(p3a, p2, descr=nextdescr) # p3a.other = p2a escape_n(p3anew) jump(p3anew) """ @@ -2819,10 +2819,10 @@ guard_nonnull(p1) [] guard_class(p1, ConstClass(node_vtable2)) [] p2 = getfield_gc_r(p1, descr=nextdescr) - guard_nonnull(12) [] + guard_nonnull(ConstPtr(nodeaddr)) [] guard_class(p2, ConstClass(node_vtable)) [] p3 = getfield_gc_r(p1, descr=otherdescr) - guard_nonnull(12) [] + guard_nonnull(ConstPtr(nodeaddr)) [] guard_class(p3, ConstClass(node_vtable)) [] p1a = new_with_vtable(descr=nodesize2) p2a = new_with_vtable(descr=nodesize) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -26,12 +26,22 @@ return op return preamble_op - def setinfo_from_preamble(self, op, preamble_info): + def setinfo_from_preamble_list(self, lst, infos): + for item in lst: + if item is None: + continue + i = infos.get(item, None) + if i is not None: + self.setinfo_from_preamble(item, i, infos) + + def setinfo_from_preamble(self, op, preamble_info, exported_infos): op = self.get_box_replacement(op) if isinstance(preamble_info, info.PtrInfo): if preamble_info.is_virtual(): # XXX do we want to sanitize this? op.set_forwarded(preamble_info) + self.setinfo_from_preamble_list(preamble_info.all_items(), + exported_infos) return if op.is_constant(): return # nothing we can learn @@ -245,6 +255,8 @@ for arg in end_args: infos[arg] = self.optimizer.getinfo(arg) label_args = virtual_state.make_inputargs(end_args, self.optimizer) + for arg in label_args: + infos[arg] = self.optimizer.getinfo(arg) sb = ShortBoxes() short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs, label_args) @@ -314,7 +326,8 @@ source.set_forwarded(target) info = exported_state.exported_infos.get(target, None) if info is not None: - self.optimizer.setinfo_from_preamble(source, info) + self.optimizer.setinfo_from_preamble(source, info, + exported_state.exported_infos) # import the optimizer state, starting from boxes that can be produced # by short preamble self.short_preamble_producer = ShortPreambleBuilder( From noreply at buildbot.pypy.org Tue Aug 11 16:56:50 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:50 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: emit guard_no_overflow in the short preamble Message-ID: <20150811145650.A208C1C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78911:c4b254735dd6 Date: 2015-08-11 15:14 +0200 http://bitbucket.org/pypy/pypy/changeset/c4b254735dd6/ Log: emit guard_no_overflow in the short preamble diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -333,6 +333,9 @@ else: self._items = [None] * size + def all_items(self): + return self._items + def _force_elements(self, op, optforce, descr): arraydescr = op.getdescr() for i in range(self.length): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -64,8 +64,11 @@ op = self.get_box_replacement(op) if op.is_constant(): return - assert op.get_forwarded() is None - op.set_forwarded(bound) + cur = op.get_forwarded() + if cur is not None: + cur.intersect(bound) + else: + op.set_forwarded(bound) def getnullness(self, op): if op.type == 'i': diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -273,12 +273,14 @@ arg.set_forwarded(None) #self.force_info_from(arg) <- XXX? self.short.append(preamble_op) + if preamble_op.is_ovf(): + self.short.append(ResOperation(rop.GUARD_NO_OVERFLOW, [], None)) info = preamble_op.get_forwarded() preamble_op.set_forwarded(None) if info is not empty_info: info.make_guards(preamble_op, self.short) if optimizer is not None: - optimizer.setinfo_from_preamble(box, info) + optimizer.setinfo_from_preamble(box, info, None) return preamble_op def add_preamble_op(self, preamble_op): From noreply at buildbot.pypy.org Tue Aug 11 16:56:52 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:52 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix this test Message-ID: <20150811145652.BF7D31C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78912:ee4ba9d13eb1 Date: 2015-08-11 15:57 +0200 http://bitbucket.org/pypy/pypy/changeset/ee4ba9d13eb1/ Log: fix this test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3297,40 +3297,40 @@ def test_ovf_guard_in_short_preamble2(self): ops = """ [p8, p11, p12] - p16 = getfield_gc_r(p8, descr=valuedescr) - i17 = getfield_gc_i(p8, descr=nextdescr) + p16 = getfield_gc_r(p8, descr=nextdescr) + i17 = getfield_gc_i(p8, descr=valuedescr) i19 = getfield_gc_i(p16, descr=valuedescr) i20 = int_ge(i17, i19) guard_false(i20) [] - i21 = getfield_gc_i(p16, descr=otherdescr) - i22 = getfield_gc_i(p16, descr=nextdescr) + i21 = getfield_gc_i(p16, descr=chardescr) + i22 = getfield_gc_i(p16, descr=valuedescr) i23 = int_mul(i17, i22) i24 = int_add(i21, i23) - p26 = new_with_vtable(descr=nodesize) + p26 = new(descr=ssize) setfield_gc(p26, i24, descr=adescr) i28 = int_add(i17, 1) - setfield_gc(p8, i28, descr=nextdescr) - i34 = getfield_gc_pure_i(p11, descr=valuedescr) + setfield_gc(p8, i28, descr=valuedescr) + i34 = getfield_gc_pure_i(p11, descr=valuedescr3) i35 = getfield_gc_pure_i(p26, descr=adescr) guard_nonnull(p12) [] i36 = int_add_ovf(i34, i35) guard_no_overflow() [] - p38 = new_with_vtable(descr=nodesize) + p38 = new(descr=ssize) setfield_gc(p38, i36, descr=adescr) jump(p8, p11, p26) """ expected = """ - [p8, p11, i24, i39, i19, p16, i21, i34] - i40 = int_ge(i39, i19) - guard_false(i40) [] - i41 = getfield_gc_i(p16, descr=nextdescr) - i42 = int_mul(i39, i41) - i43 = int_add(i21, i42) - i44 = int_add(i39, 1) - setfield_gc(p8, i44, descr=nextdescr) - i45 = int_add_ovf(i34, i43) + [p0, p1, i9, p3, i10, i7, i11] + i13 = getfield_gc_i(p3, descr=valuedescr) + i14 = int_ge(i10, i13) + guard_false(i14) [] + i15 = int_mul(i10, i13) + i16 = int_add(i7, i15) + i17 = int_add(i10, 1) + setfield_gc(p0, i17, descr=valuedescr) + i18 = int_add_ovf(i11, i16) guard_no_overflow() [] - jump(p8, p11, i43, i44, i19, p16, i21, i34) + jump(p0, p1, i16, p3, i17, i7, i11) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -34,6 +34,12 @@ if i is not None: self.setinfo_from_preamble(item, i, infos) + def is_inputarg(self, op): + if self.optunroll.short_preamble_producer is None: + return op in self.inparg_dict + return (op in self.inparg_dict or + op in self.optunroll.short_preamble_producer.used_boxes) + def setinfo_from_preamble(self, op, preamble_info, exported_infos): op = self.get_box_replacement(op) if isinstance(preamble_info, info.PtrInfo): @@ -71,6 +77,7 @@ distinction anymore)""" ops_to_import = None + short_preamble_producer = None def __init__(self, metainterp_sd, jitdriver_sd, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd, From noreply at buildbot.pypy.org Tue Aug 11 16:56:54 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:54 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: hack hack hack Message-ID: <20150811145654.DFCEB1C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78913:838e242ff7a9 Date: 2015-08-11 16:38 +0200 http://bitbucket.org/pypy/pypy/changeset/838e242ff7a9/ Log: hack hack hack diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -146,8 +146,11 @@ old_op = self.optimizer._newoperations[pos] if self.optimize_call_pure(op, old_op): return - for old_op in self.extra_call_pure: + for i, old_op in enumerate(self.extra_call_pure): if self.optimize_call_pure(op, old_op): + if isinstance(old_op, PreambleOp): + old_op = self.optimizer.force_op_from_preamble(old_op) + self.extra_call_pure[i] = old_op return # replace CALL_PURE with just CALL @@ -181,6 +184,9 @@ # all identical # this removes a CALL_PURE that has the same (non-constant) # arguments as a previous CALL_PURE. + if isinstance(old_op, PreambleOp): + # xxx obscure, it's dealt with in the caller + old_op = old_op.op self.make_equal_to(op, old_op) self.last_emitted_operation = REMOVED return True @@ -226,11 +232,9 @@ if op.is_ovf() and ops[i + 1].getopnum() == rop.GUARD_NO_OVERFLOW: sb.add_pure_op(op) for i in self.call_pure_positions: - yyy op = ops[i] - assert op.getopnum() == rop.CALL - op = op.copy_and_change(rop.CALL_PURE) - sb.add_potential(op) + assert op.is_call() + sb.add_pure_op(op) dispatch_opt = make_dispatcher_method(OptPure, 'optimize_', default=OptPure.optimize_default) diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -20,6 +20,12 @@ self.op = op self.preamble_op = preamble_op + def numargs(self): + return self.op.numargs() + + def getarglist(self): + return self.op.getarglist() + def getarg(self, i): return self.op.getarg(i) @@ -68,7 +74,10 @@ if optpure is None: return op = self.res - opt.pure(op.getopnum(), PreambleOp(op, preamble_op)) + if op.is_call(): + optpure.extra_call_pure.append(PreambleOp(op, preamble_op)) + else: + opt.pure(op.getopnum(), PreambleOp(op, preamble_op)) def __repr__(self): return "PureOp(%r)" % (self.res,) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3653,7 +3653,6 @@ setfield_gc(p1, i1, descr=valuedescr) i3 = call_assembler_i(i1, descr=asmdescr) setfield_gc(p1, i3, descr=valuedescr) - i143 = same_as(i3) # Should be killed by backend jump(p1, i4, i3) ''' self.optimize_loop(ops, ops, preamble) @@ -3678,18 +3677,16 @@ jump(p1, i4, i3) ''' expected = ''' - [p1, i4, i3, i5] - setfield_gc(p1, i5, descr=valuedescr) - jump(p1, i3, i5, i5) + [p1, i4, i3] + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i3, i3) ''' preamble = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call_i(p1, descr=elidablecalldescr) setfield_gc(p1, i3, descr=valuedescr) - i148 = same_as(i3) - i147 = same_as(i3) - jump(p1, i4, i3, i148) + jump(p1, i4, i3) ''' self.optimize_loop(ops, expected, preamble) @@ -3706,9 +3703,9 @@ jump(p1, i4, i3) ''' expected = ''' - [p1, i4, i3, i5] - setfield_gc(p1, i5, descr=valuedescr) - jump(p1, i3, i5, i5) + [p1, i4, i3] + setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i3, i3) ''' preamble = ''' [p1, i1, i4] @@ -3716,9 +3713,9 @@ i3 = call_i(p1, descr=elidable2calldescr) guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) - i148 = same_as(i3) - i147 = same_as(i3) - jump(p1, i4, i3, i148) + #i148 = same_as(i3) + #i147 = same_as(i3) + jump(p1, i4, i3) #, i148) ''' self.optimize_loop(ops, expected, preamble) @@ -3957,7 +3954,7 @@ [p0, i1] p3 = force_token() # - p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + p2 = new_with_vtable(descr=vref_descr) setfield_gc(p2, NULL, descr=virtualforceddescr) setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -21,6 +21,7 @@ def force_op_from_preamble(self, preamble_op): if isinstance(preamble_op, PreambleOp): op = preamble_op.op + self.optimizer.inparg_dict[op] = None # XXX ARGH self.optunroll.short_preamble_producer.use_box(op, self) self.optunroll.potential_extra_ops[op] = preamble_op return op @@ -34,12 +35,6 @@ if i is not None: self.setinfo_from_preamble(item, i, infos) - def is_inputarg(self, op): - if self.optunroll.short_preamble_producer is None: - return op in self.inparg_dict - return (op in self.inparg_dict or - op in self.optunroll.short_preamble_producer.used_boxes) - def setinfo_from_preamble(self, op, preamble_info, exported_infos): op = self.get_box_replacement(op) if isinstance(preamble_info, info.PtrInfo): @@ -76,7 +71,6 @@ become the preamble or entry bridge (don't think there is a distinction anymore)""" - ops_to_import = None short_preamble_producer = None def __init__(self, metainterp_sd, jitdriver_sd, optimizations): From noreply at buildbot.pypy.org Tue Aug 11 16:56:57 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:57 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix some tests Message-ID: <20150811145657.198B81C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78914:3abe6c76a02f Date: 2015-08-11 16:47 +0200 http://bitbucket.org/pypy/pypy/changeset/3abe6c76a02f/ Log: fix some tests diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7983,15 +7983,15 @@ ops = """ [p5, p6] i10 = getfield_gc_i(p5, descr=valuedescr) - i11 = getfield_gc_i(p6, descr=nextdescr) + i11 = getfield_gc_i(p6, descr=chardescr) call_n(i10, i11, descr=nonwritedescr) - setfield_gc(p6, i10, descr=nextdescr) + setfield_gc(p6, i10, descr=chardescr) jump(p5, p6) """ expected = """ [p5, p6, i10, i11] call_n(i10, i11, descr=nonwritedescr) - setfield_gc(p6, i10, descr=nextdescr) + setfield_gc(p6, i10, descr=chardescr) jump(p5, p6, i10, i10) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py --- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py @@ -291,4 +291,4 @@ jump(p1) """ es, loop, preamble = self.optimize(ops) - xxx + # assert did not crash From noreply at buildbot.pypy.org Tue Aug 11 16:56:59 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:56:59 +0200 (CEST) Subject: [pypy-commit] pypy default: reenable the test Message-ID: <20150811145659.252691C0206@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r78915:df812fe311a7 Date: 2015-08-11 16:54 +0200 http://bitbucket.org/pypy/pypy/changeset/df812fe311a7/ Log: reenable the test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7352,7 +7352,9 @@ i3 = int_add(i1, i2) setfield_gc(p0, ii, descr=valuedescr) setfield_gc(p1, ii, descr=otherdescr) - jump(p0, p1, ii2, ii, ii, ii) + i7 = same_as(ii) + i8 = same_as(ii) + jump(p0, p1, ii2, ii, i8, i7) """ expected = """ [p0, p1, ii, ii2, i1, i2] @@ -7361,7 +7363,7 @@ setfield_gc(p1, ii, descr=otherdescr) jump(p0, p1, ii2, ii, ii, ii) """ - self.optimize_loop(ops, expected) + self.optimize_loop(ops, expected, preamble) def test_dont_specialize_on_boxes_equal(self): ops = """ From noreply at buildbot.pypy.org Tue Aug 11 16:57:01 2015 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 11 Aug 2015 16:57:01 +0200 (CEST) Subject: [pypy-commit] pypy default: merge Message-ID: <20150811145701.4AA5C1C04C1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r78916:af4f9dce75b8 Date: 2015-08-11 16:56 +0200 http://bitbucket.org/pypy/pypy/changeset/af4f9dce75b8/ Log: merge diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -416,6 +416,9 @@ arg_driver = jit.JitDriver(name='numpy_' + op_name, greens = ['shapelen', 'dtype'], reds = 'auto') + arg_flat_driver = jit.JitDriver(name='numpy_flat_' + op_name, + greens = ['shapelen', 'dtype'], + reds = 'auto') def argmin_argmax(space, w_arr, w_out, axis): from pypy.module.micronumpy.descriptor import get_dtype_cache @@ -457,7 +460,7 @@ state = iter.next(state) shapelen = len(w_arr.get_shape()) while not iter.done(state): - arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) + arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) w_val = iter.getitem(state) new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) if dtype.itemtype.ne(new_best, cur_best): diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -425,8 +425,9 @@ class __extend__(pairtype(SomeString, SomeObject), pairtype(SomeUnicodeString, SomeObject)): - def mod((s_string, args)): - return s_string.__class__() + def mod((s_string, s_arg)): + assert not isinstance(s_arg, SomeTuple) + return pair(s_string, SomeTuple([s_arg])).mod() class __extend__(pairtype(SomeFloat, SomeFloat)): diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2124,6 +2124,16 @@ assert isinstance(s, annmodel.SomeString) assert s.no_nul + def test_no_nul_mod(self): + def f(x): + s = "%d" % x + return s + a = self.RPythonAnnotator() + s = a.build_types(f, [int]) + assert isinstance(s, annmodel.SomeString) + assert s.no_nul + + def test_mul_str0(self): def f(s): return s*10 diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -79,6 +79,9 @@ assert methname == 'jit_merge_point', ( "reds='auto' is supported only for jit drivers which " "calls only jit_merge_point. Found a call to %s" % methname) + if jitdriver.numreds is not None: + raise AssertionError("there are multiple jit_merge_points " + "with the same jitdriver") # # compute the set of live variables across the jit_marker alive_v = set() @@ -96,10 +99,7 @@ v.concretetype is not lltype.Void] reds_v = sort_vars(reds_v) op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + jitdriver.numreds = len(reds_v) def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -558,6 +558,22 @@ assert res == 7 - 3 self.check_trace_count(2) + def test_jitdriver_single_jit_merge_point(self): + jitdriver = JitDriver(greens=[], reds='auto') + def g1(n): + jitdriver.jit_merge_point() + return n + def g2(): + jitdriver.jit_merge_point() + def f(n): + if n: + g1(n) + else: + g2() + e = py.test.raises(AssertionError, self.meta_interp, f, [42]) + assert str(e.value) == ("there are multiple jit_merge_points " + "with the same jitdriver") + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU diff --git a/rpython/rlib/debug.py b/rpython/rlib/debug.py --- a/rpython/rlib/debug.py +++ b/rpython/rlib/debug.py @@ -442,19 +442,26 @@ eci = ExternalCompilationInfo(includes=['string.h', 'assert.h', 'sys/prctl.h'], post_include_bits=[""" +/* If we have an old Linux kernel (or compile with old system headers), + the following two macros are not defined. But we would still like + a pypy translated on such a system to run on a more modern system. */ +#ifndef PR_SET_PTRACER +# define PR_SET_PTRACER 0x59616d61 +#endif +#ifndef PR_SET_PTRACER_ANY +# define PR_SET_PTRACER_ANY ((unsigned long)-1) +#endif static void pypy__allow_attach(void) { prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY); - return; } """]) + allow_attach = rffi.llexternal( + "pypy__allow_attach", [], lltype.Void, + compilation_info=eci, _nowrapper=True) else: # Do nothing, there's no prctl - eci = ExternalCompilationInfo(post_include_bits=[ - "static void pypy__allow_attach(void) { return; }"]) - - allow_attach = rffi.llexternal( - "pypy__allow_attach", [], lltype.Void, - compilation_info=eci, _nowrapper=True) + def allow_attach(): + pass def impl_attach_gdb(): import os diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -806,10 +806,14 @@ flatten = {} caller = sys._getframe(1) caller_name = caller.f_globals.get('__name__') + immutable_fields = [] for base in inspect.getmro(M): if base is object: continue for key, value in base.__dict__.items(): + if key == '_immutable_fields_': + immutable_fields.extend(value) + continue if key.startswith('__') and key.endswith('__'): if key not in special_methods: continue @@ -835,4 +839,9 @@ if key in target: raise Exception("import_from_mixin: would overwrite the value " "already defined locally for %r" % (key,)) + if key == '_mixin_': + raise Exception("import_from_mixin(M): class M should not " + "have '_mixin_ = True'") target[key] = value + if immutable_fields: + target['_immutable_fields_'] = target.get('_immutable_fields_', []) + immutable_fields diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -675,3 +675,44 @@ import_from_mixin(M) assert A.f is not M.f assert A.f.__module__ != M.f.__module__ + + +def test_import_from_mixin_immutable_fields(): + class A(object): + _immutable_fields_ = ['a'] + + class B(object): + _immutable_fields_ = ['b'] + import_from_mixin(A) + + assert B._immutable_fields_ == ['b', 'a'] + assert A._immutable_fields_ == ['a'] + + + class B(object): + import_from_mixin(A) + + assert B._immutable_fields_ == ['a'] + + class C(A): + _immutable_fields_ = ['c'] + + + class B(object): + import_from_mixin(C) + + assert B._immutable_fields_ == ['c', 'a'] + + class B(object): + _immutable_fields_ = ['b'] + import_from_mixin(C) + + assert B._immutable_fields_ == ['b', 'c', 'a'] + + + class B(object): + _immutable_fields_ = ['b'] + class BA(B): + import_from_mixin(C) + + assert BA._immutable_fields_ == ['c', 'a'] From noreply at buildbot.pypy.org Tue Aug 11 17:10:15 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 17:10:15 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): more printing Message-ID: <20150811151015.C72071C04C1@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78917:91a56056879f Date: 2015-08-11 17:09 +0200 http://bitbucket.org/pypy/pypy/changeset/91a56056879f/ Log: (arigo, cfbolz): more printing diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -38,6 +38,8 @@ self._vprof_status = SEEN_INT elif status == SEEN_INT: if self.read_constant_int() != value: + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT", self._vprof_msg, 'int', value self._vprof_status = SEEN_TOO_MUCH else: self._vprof_counter += 1 @@ -45,6 +47,8 @@ print self._vprof_msg, 'int', value elif status == SEEN_OBJ: self._vprof_status = SEEN_TOO_MUCH + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT", self._vprof_msg, 'int', value def see_object(self, value): status = self._vprof_status @@ -59,9 +63,13 @@ # for tests, which really use unwrapped ints in a few places self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_INT: + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_OBJ: if self.try_read_constant_obj() is not value: + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value self._vprof_status = SEEN_TOO_MUCH else: self._vprof_counter += 1 From noreply at buildbot.pypy.org Tue Aug 11 17:10:18 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 17:10:18 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): another place where locals_cells_stack_w is written to Message-ID: <20150811151018.01D221C04C1@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78918:d9ccb4d048c0 Date: 2015-08-11 17:10 +0200 http://bitbucket.org/pypy/pypy/changeset/d9ccb4d048c0/ Log: (arigo, cfbolz): another place where locals_cells_stack_w is written to diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py --- a/pypy/interpreter/function.py +++ b/pypy/interpreter/function.py @@ -105,7 +105,9 @@ self) for i in funccallunrolling: if i < nargs: - new_frame.locals_cells_stack_w[i] = args_w[i] + w_arg = args_w[i] + new_frame.locals_cells_stack_w[i] = w_arg + new_frame._value_profile_local(i, w_arg) return new_frame.run() elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -536,3 +536,17 @@ res = f(10).g() sys.settrace(None) assert res == 10 + +class TestValueProf(object): + def test_argument_is_constant(self): + space = self.space + w_f = space.appexec([], """(): + def f(x): + y = x + 1 + return f""") + import pdb; pdb.set_trace() + space.call_function(w_f, space.wrap(1)) + assert len(w_f.code.vprofs) == 2 + assert w_f.code.vprofs[0].can_fold_read_int() + assert w_f.code.vprofs[1].can_fold_read_int() + From noreply at buildbot.pypy.org Tue Aug 11 18:12:44 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 18:12:44 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: pdb.set_trace() not expected any more Message-ID: <20150811161244.338571C04C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: value-profiling Changeset: r78919:b9231d39eb6a Date: 2015-08-11 17:09 +0100 http://bitbucket.org/pypy/pypy/changeset/b9231d39eb6a/ Log: pdb.set_trace() not expected any more diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -544,7 +544,6 @@ def f(x): y = x + 1 return f""") - import pdb; pdb.set_trace() space.call_function(w_f, space.wrap(1)) assert len(w_f.code.vprofs) == 2 assert w_f.code.vprofs[0].can_fold_read_int() From noreply at buildbot.pypy.org Tue Aug 11 18:12:46 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 18:12:46 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: Add some missing see_write()s Message-ID: <20150811161246.550CA1C04C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: value-profiling Changeset: r78920:fb8245228f45 Date: 2015-08-11 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/fb8245228f45/ Log: Add some missing see_write()s diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py --- a/pypy/interpreter/function.py +++ b/pypy/interpreter/function.py @@ -106,8 +106,7 @@ for i in funccallunrolling: if i < nargs: w_arg = args_w[i] - new_frame.locals_cells_stack_w[i] = w_arg - new_frame._value_profile_local(i, w_arg) + new_frame._setlocal(i, w_arg) return new_frame.run() elif nargs >= 1 and fast_natural_arity == Code.PASSTHROUGHARGS1: assert isinstance(code, gateway.BuiltinCodePassThroughArguments1) @@ -173,7 +172,7 @@ self) for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) - new_frame.locals_cells_stack_w[i] = w_arg + new_frame._setlocal(i, w_arg) return new_frame.run() @@ -184,13 +183,13 @@ self) for i in xrange(nargs): w_arg = frame.peekvalue(nargs-1-i) - new_frame.locals_cells_stack_w[i] = w_arg + new_frame._setlocal(i, w_arg) ndefs = len(self.defs_w) start = ndefs - defs_to_load i = nargs for j in xrange(start, ndefs): - new_frame.locals_cells_stack_w[i] = self.defs_w[j] + new_frame._setlocal(i, self.defs_w[j]) i += 1 return new_frame.run() diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -216,9 +216,7 @@ fresh_virtualizable=True) args.parse_into_scope(None, fresh_frame.locals_cells_stack_w, func.name, sig, func.defs_w) - for arg in range(self.co_nlocals): - fresh_frame._value_profile_local( - arg, fresh_frame.locals_cells_stack_w[arg]) + fresh_frame._all_locals_changed() fresh_frame.init_cells() return frame.run() @@ -232,9 +230,7 @@ fresh_virtualizable=True) args.parse_into_scope(w_obj, fresh_frame.locals_cells_stack_w, func.name, sig, func.defs_w) - for arg in range(self.co_nlocals): - fresh_frame._value_profile_local( - arg, fresh_frame.locals_cells_stack_w[arg]) + fresh_frame._all_locals_changed() fresh_frame.init_cells() return frame.run() diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -168,6 +168,10 @@ vprof = self.getcode().vprofs[varindex] vprof.see_write(value) + def _all_locals_changed(self): + for i, vprof in enumerate(self.getcode().vprofs): + vprof.see_write(self.locals_cells_stack_w[i]) + def mark_as_escaped(self): """ Must be called on frames that are exposed to applevel, e.g. by @@ -525,6 +529,7 @@ new_frame.set_blocklist([unpickle_block(space, w_blk) for w_blk in space.unpackiterable(w_blockstack)]) self.locals_cells_stack_w = values_w[:] + self._all_locals_changed() valuestackdepth = space.int_w(w_stackdepth) if not self._check_stack_index(valuestackdepth): raise OperationError(space.w_ValueError, space.wrap("invalid stackdepth")) From noreply at buildbot.pypy.org Tue Aug 11 18:18:39 2015 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 11 Aug 2015 18:18:39 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: pom pom pom Message-ID: <20150811161839.9280F1C04C1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: value-profiling Changeset: r78921:bdc51d0e73ec Date: 2015-08-11 17:18 +0100 http://bitbucket.org/pypy/pypy/changeset/bdc51d0e73ec/ Log: pom pom pom diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -35,10 +35,10 @@ w_args, w_kwds = __args__.topacked() bottomframe = space.createframe(get_entrypoint_pycode(space), get_w_module_dict(space), None) - bottomframe.locals_cells_stack_w[0] = space.wrap(self) - bottomframe.locals_cells_stack_w[1] = w_callable - bottomframe.locals_cells_stack_w[2] = w_args - bottomframe.locals_cells_stack_w[3] = w_kwds + bottomframe._setlocal(0, space.wrap(self)) + bottomframe._setlocal(1, w_callable) + bottomframe._setlocal(2, w_args) + bottomframe._setlocal(3, w_kwds) bottomframe.last_exception = get_cleared_operation_error(space) self.bottomframe = bottomframe # From noreply at buildbot.pypy.org Tue Aug 11 18:20:54 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 18:20:54 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: passing None to space.call_method is a bad idea Message-ID: <20150811162054.1E72C1C04C1@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78922:5ba373083b5c Date: 2015-08-11 18:20 +0200 http://bitbucket.org/pypy/pypy/changeset/5ba373083b5c/ Log: passing None to space.call_method is a bad idea diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py --- a/pypy/module/micronumpy/ufuncs.py +++ b/pypy/module/micronumpy/ufuncs.py @@ -313,7 +313,7 @@ out.implementation.setitem(0, res) res = out if call__array_wrap__: - res = space.call_method(obj, '__array_wrap__', res, None) + res = space.call_method(obj, '__array_wrap__', res, space.w_None) return res else: @@ -360,7 +360,7 @@ loop.reduce( space, self.func, obj, axis_flags, dtype, out, self.identity) if call__array_wrap__: - out = space.call_method(obj, '__array_wrap__', out, None) + out = space.call_method(obj, '__array_wrap__', out, space.w_None) return out def descr_outer(self, space, __args__): From noreply at buildbot.pypy.org Tue Aug 11 18:53:10 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 18:53:10 +0200 (CEST) Subject: [pypy-commit] pypy typed-cells: (cfbolz, arigo): small code simplification Message-ID: <20150811165310.78CF41C0683@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: typed-cells Changeset: r78924:8c68aa37e287 Date: 2015-08-11 18:53 +0200 http://bitbucket.org/pypy/pypy/changeset/8c68aa37e287/ Log: (cfbolz, arigo): small code simplification diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -306,24 +306,30 @@ from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.floatobject import W_FloatObject assert not isinstance(w_cell, ObjectMutableCell) - if isinstance(w_cell, IntMutableCell) and type(w_value) is W_IntObject: - w_cell.intvalue = w_value.intval - return None - if isinstance(w_cell, FloatMutableCell) and type(w_value) is W_FloatObject: - w_cell.floatvalue = w_value.floatval - return None if type(w_value) is W_IntObject: - if not self.can_contain_mutable_cell: - self.can_contain_mutable_cell = True + if isinstance(w_cell, IntMutableCell): + w_cell.intvalue = w_value.intval + return None + check = self._ensure_can_contain_mutable_cell() + assert check if self.ever_mutated: return IntMutableCell(w_value.intval) if type(w_value) is W_FloatObject: - if not self.can_contain_mutable_cell: - self.can_contain_mutable_cell = True + if isinstance(w_cell, FloatMutableCell): + w_cell.floatvalue = w_value.floatval + return None + check = self._ensure_can_contain_mutable_cell() + assert check if self.ever_mutated: return FloatMutableCell(w_value.floatval) return w_value + @jit.elidable + def _ensure_can_contain_mutable_cell(self): + if not self.can_contain_mutable_cell: + self.can_contain_mutable_cell = True + return True + def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.selector) new_obj._get_mapdict_map().add_attr(new_obj, self.selector, w_value) From noreply at buildbot.pypy.org Tue Aug 11 19:11:43 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 19:11:43 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (cfbolz, arigo): interpreter optimization Message-ID: <20150811171143.7B0051C04C1@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78925:c7051e9e1c11 Date: 2015-08-11 19:11 +0200 http://bitbucket.org/pypy/pypy/changeset/c7051e9e1c11/ Log: (cfbolz, arigo): interpreter optimization diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -1,4 +1,4 @@ -from rpython.rlib import jit +from rpython.rlib import jit, objectmodel from rpython.rlib.rweakref import ref, dead_ref SEEN_NOTHING = '\x00' @@ -26,7 +26,10 @@ def get_int_val(self, w_obj): raise NotImplementedError("abstract base") + @objectmodel.always_inline def see_write(self, w_value): + if self._vprof_status == SEEN_TOO_MUCH: + return if self.is_int(w_value): return self.see_int(self.get_int_val(w_value)) return self.see_object(w_value) From noreply at buildbot.pypy.org Tue Aug 11 19:11:45 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 19:11:45 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): make tests compatible with printing Message-ID: <20150811171145.ABA391C04C1@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78926:5a2800b3e912 Date: 2015-08-11 19:11 +0200 http://bitbucket.org/pypy/pypy/changeset/5a2800b3e912/ Log: (arigo, cfbolz): make tests compatible with printing diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -14,6 +14,7 @@ space.config = Config class Class(object): + name = 'testing' def __init__(self, hasdict=True): self.hasdict = True if hasdict: @@ -33,7 +34,7 @@ hasdict = False def test_plain_attribute(): - w_cls = "class" + w_cls = None aa = PlainAttribute(("b", DICT), PlainAttribute(("a", DICT), Terminator(space, w_cls))) @@ -61,7 +62,7 @@ assert aa.get_terminator() is aa.back.back def test_huge_chain(): - current = Terminator(space, "cls") + current = Terminator(space, None) for i in range(20000): current = PlainAttribute((str(i), DICT), current) assert current.find_map_attr(("0", DICT)).storageindex == 0 From noreply at buildbot.pypy.org Tue Aug 11 19:16:57 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 11 Aug 2015 19:16:57 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): print not when jitting Message-ID: <20150811171657.3E6421C04C1@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78927:47d8cc161fcd Date: 2015-08-11 19:17 +0200 http://bitbucket.org/pypy/pypy/changeset/47d8cc161fcd/ Log: (arigo, cfbolz): print not when jitting diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -45,9 +45,10 @@ print "NO LONGER CONSTANT", self._vprof_msg, 'int', value self._vprof_status = SEEN_TOO_MUCH else: - self._vprof_counter += 1 - if self._vprof_counter == 200: - print self._vprof_msg, 'int', value + if not jit.we_are_jitted(): + self._vprof_counter += 1 + if self._vprof_counter == 200: + print self._vprof_msg, 'int', value elif status == SEEN_OBJ: self._vprof_status = SEEN_TOO_MUCH if self._vprof_counter >= 200: @@ -75,9 +76,10 @@ print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value self._vprof_status = SEEN_TOO_MUCH else: - self._vprof_counter += 1 - if self._vprof_counter == 200: - print self._vprof_msg, 'obj', value + if not jit.we_are_jitted(): + self._vprof_counter += 1 + if self._vprof_counter == 200: + print self._vprof_msg, 'obj', value def can_fold_read_int(self): return self._vprof_status == SEEN_INT From noreply at buildbot.pypy.org Tue Aug 11 19:18:35 2015 From: noreply at buildbot.pypy.org (plan_rich) Date: Tue, 11 Aug 2015 19:18:35 +0200 (CEST) Subject: [pypy-commit] pypy vecopt-merge: started to refactor scheduling to remove the unnecessary list allocation each iteration Message-ID: <20150811171835.6867B1C04C1@cobra.cs.uni-duesseldorf.de> Author: Richard Plangger Branch: vecopt-merge Changeset: r78928:8bbff2ecdaa3 Date: 2015-08-11 19:18 +0200 http://bitbucket.org/pypy/pypy/changeset/8bbff2ecdaa3/ Log: started to refactor scheduling to remove the unnecessary list allocation each iteration diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py --- a/rpython/jit/metainterp/optimizeopt/schedule.py +++ b/rpython/jit/metainterp/optimizeopt/schedule.py @@ -17,26 +17,13 @@ self.graph = graph self.schedulable_nodes = self.graph.schedulable_nodes self.sched_data = sched_data + self.oplist = None + self.renamer = None def has_more(self): return len(self.schedulable_nodes) > 0 - def next(self, renamer, position): - i = self._next(self.schedulable_nodes) - if i >= 0: - candidate = self.schedulable_nodes[i] - del self.schedulable_nodes[i] - return self.schedule(candidate, renamer, position) - - # it happens that packs can emit many nodes that have been - # added to the scheuldable_nodes list, in this case it could - # be that no next exists even though the list contains elements - if not self.has_more(): - return [] - - raise AssertionError("schedule failed cannot continue. possible reason: cycle") - - def _next(self, candidate_list): + def next_index(self, candidate_list): i = len(candidate_list)-1 while i >= 0: candidate = candidate_list[i] @@ -64,22 +51,8 @@ return False return candidate.depends_count() == 0 - def schedule(self, candidate, renamer, position): - if candidate.pack: - pack = candidate.pack - for node in pack.operations: - renamer.rename(node.getoperation()) - vops = self.sched_data.as_vector_operation(pack, renamer) - for node in pack.operations: - self.scheduled(node, position) - return vops - else: - self.scheduled(candidate, position) - renamer.rename(candidate.getoperation()) - return [candidate.getoperation()] - - def scheduled(self, node, position): - node.position = position + def scheduled(self, node): + node.position = len(self.oplist) for dep in node.provides()[:]: # COPY to = dep.to node.remove_edge_to(to) @@ -105,6 +78,26 @@ node.clear_dependencies() node.emitted = True + def emit_into(self, oplist, renamer, unpack=False): + self.renamer = renamer + self.oplist = oplist + self.unpack = unpack + while self.has_more(): + i = self.next_index(self.schedulable_nodes) + if i >= 0: + candidate = self.schedulable_nodes[i] + del self.schedulable_nodes[i] + self.sched_data.schedule_candidate(self, candidate) + continue + + # it happens that packs can emit many nodes that have been + # added to the scheuldable_nodes list, in this case it could + # be that no next exists even though the list contains elements + if not self.has_more(): + break + + raise AssertionError("schedule failed cannot continue. possible reason: cycle") + def vectorbox_outof_box(box, count=-1, size=-1, type='-'): if box.type not in (FLOAT, INT): raise AssertionError("cannot create vector box of type %s" % (box.type)) @@ -251,22 +244,6 @@ self.output_type = None self.costmodel = None - def check_if_pack_supported(self, pack): - op0 = pack.operations[0].getoperation() - if self.input_type is None: - # must be a load/guard op - return - insize = self.input_type.getsize() - if op0.casts_box(): - # prohibit the packing of signext calls that - # cast to int16/int8. - _, outsize = op0.cast_to() - self.sched_data._prevent_signext(outsize, insize) - if op0.getopnum() == rop.INT_MUL: - if insize == 8 or insize == 1: - # see assembler for comment why - raise NotAProfitableLoop - def as_vector_operation(self, pack, sched_data, oplist): self.sched_data = sched_data self.vecops = oplist @@ -298,6 +275,23 @@ def before_argument_transform(self, args): pass + def check_if_pack_supported(self, pack): + op0 = pack.operations[0].getoperation() + if self.input_type is None: + # must be a load/guard op + return + insize = self.input_type.getsize() + if op0.casts_box(): + # prohibit the packing of signext calls that + # cast to int16/int8. + _, outsize = op0.cast_to() + self.sched_data._prevent_signext(outsize, insize) + if op0.getopnum() == rop.INT_MUL: + if insize == 8 or insize == 1: + # see assembler for comment why + raise NotAProfitableLoop + + def transform_pack(self): op = self.pack.leftmost() args = op.getarglist() @@ -754,30 +748,89 @@ self.inputargs[arg] = None self.seen = {} - def _prevent_signext(self, outsize, insize): - if insize != outsize: - if outsize < 4 or insize < 4: - raise NotAProfitableLoop + def schedule_candidate(self, scheduler, candidate): + """ if you implement a scheduler this operations is called + to emit the actual operation into the oplist of the scheduler + """ + renamer = scheduler.renamer + if candidate.pack: + for node in candidate.pack.operations: + self.unpack_from_vector(candidate.getoperation(), renamer) + scheduler.scheduled(node) + #renamer.rename(node.getoperation()) + self.as_vector_operation(scheduler, candidate.pack) + else: + self.unpack_from_vector(candidate.getoperation(), renamer) + scheduler.scheduled(candidate) + #renamer.rename(candidate.getoperation()) + op = candidate.getoperation() + scheduler.oplist.append(op) - def as_vector_operation(self, pack, preproc_renamer): + def as_vector_operation(self, scheduler, pack): assert pack.opcount() > 1 # properties that hold for the pack are: # + isomorphism (see func above) # + tightly packed (no room between vector elems) - oplist = [] + oplist = scheduler.oplist + position = len(oplist) op = pack.operations[0].getoperation() determine_trans(op).as_vector_operation(pack, self, oplist) # + # XXX if pack.is_accumulating(): - box = oplist[0].result + box = oplist[position].result assert box is not None for node in pack.operations: op = node.getoperation() assert op.result is not None preproc_renamer.start_renaming(op.result, box) - # - return oplist + + def unpack_from_vector(self, op, renamer): + renamer.rename(op) + args = op.getarglist() + + # unpack for an immediate use + for i, arg in enumerate(op.getarglist()): + if isinstance(arg, Box): + argument = self._unpack_from_vector(i, arg, renamer) + if arg is not argument: + op.setarg(i, argument) + if op.result: + self.seen[op.result] = None + # unpack for a guard exit + if op.is_guard(): + fail_args = op.getfailargs() + for i, arg in enumerate(fail_args): + if arg and isinstance(arg, Box): + argument = self._unpack_from_vector(i, arg, renamer) + if arg is not argument: + fail_args[i] = argument + + def _unpack_from_vector(self, i, arg, renamer): + if arg in self.seen or arg.type == 'V': + return arg + (j, vbox) = self.getvector_of_box(arg) + if vbox: + if vbox in self.invariant_vector_vars: + return arg + arg_cloned = arg.clonebox() + self.seen[arg_cloned] = None + renamer.start_renaming(arg, arg_cloned) + self.setvector_of_box(arg_cloned, j, vbox) + cj = ConstInt(j) + ci = ConstInt(1) + opnum = getunpackopnum(vbox.gettype()) + unpack_op = ResOperation(opnum, [vbox, cj, ci], arg_cloned) + self.costmodel.record_vector_unpack(vbox, j, 1) + self.emit_operation(unpack_op) + return arg_cloned + return arg + + def _prevent_signext(self, outsize, insize): + if insize != outsize: + if outsize < 4 or insize < 4: + raise NotAProfitableLoop def getvector_of_box(self, arg): return self.box_to_vbox.get(arg, (-1, None)) diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py --- a/rpython/jit/metainterp/optimizeopt/vectorize.py +++ b/rpython/jit/metainterp/optimizeopt/vectorize.py @@ -460,13 +460,7 @@ sched_data.seen[arg] = None break # - while scheduler.has_more(): - position = len(self._newoperations) - ops = scheduler.next(renamer, position) - for op in ops: - if vector: - self.unpack_from_vector(op, sched_data, renamer) - self.emit_operation(op) + scheduler.emit_into(self._newoperations, renamer, unpack=vector) # if not we_are_translated(): for node in self.dependency_graph.nodes: @@ -490,47 +484,6 @@ self.orig_label_args) self.clear_newoperations() - def unpack_from_vector(self, op, sched_data, renamer): - renamer.rename(op) - args = op.getarglist() - - # unpack for an immediate use - for i, arg in enumerate(op.getarglist()): - if isinstance(arg, Box): - argument = self._unpack_from_vector(i, arg, sched_data, renamer) - if arg is not argument: - op.setarg(i, argument) - if op.result: - sched_data.seen[op.result] = None - # unpack for a guard exit - if op.is_guard(): - fail_args = op.getfailargs() - for i, arg in enumerate(fail_args): - if arg and isinstance(arg, Box): - argument = self._unpack_from_vector(i, arg, sched_data, renamer) - if arg is not argument: - fail_args[i] = argument - - def _unpack_from_vector(self, i, arg, sched_data, renamer): - if arg in sched_data.seen or arg.type == 'V': - return arg - (j, vbox) = sched_data.getvector_of_box(arg) - if vbox: - if vbox in sched_data.invariant_vector_vars: - return arg - arg_cloned = arg.clonebox() - sched_data.seen[arg_cloned] = None - renamer.start_renaming(arg, arg_cloned) - sched_data.setvector_of_box(arg_cloned, j, vbox) - cj = ConstInt(j) - ci = ConstInt(1) - opnum = getunpackopnum(vbox.gettype()) - unpack_op = ResOperation(opnum, [vbox, cj, ci], arg_cloned) - self.costmodel.record_vector_unpack(vbox, j, 1) - self.emit_operation(unpack_op) - return arg_cloned - return arg - def analyse_index_calculations(self): ee_pos = self.loop.find_first_index(rop.GUARD_EARLY_EXIT) if len(self.loop.operations) <= 2 or ee_pos == -1: From noreply at buildbot.pypy.org Wed Aug 12 12:02:19 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 12:02:19 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (arigo, cfbolz): fix unroll_safe Message-ID: <20150812100219.BA7401C0683@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78929:d13ecd04dfb3 Date: 2015-08-12 12:02 +0200 http://bitbucket.org/pypy/pypy/changeset/d13ecd04dfb3/ Log: (arigo, cfbolz): fix unroll_safe diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -206,7 +206,6 @@ self.fast_natural_arity = eval.Code.FLATPYCALL | self.co_argcount - @jit.unroll_safe def funcrun(self, func, args): frame = self.space.createframe(self, func.w_func_globals, func) @@ -220,7 +219,6 @@ fresh_frame.init_cells() return frame.run() - @jit.unroll_safe def funcrun_obj(self, func, w_obj, args): frame = self.space.createframe(self, func.w_func_globals, func) diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -168,6 +168,7 @@ vprof = self.getcode().vprofs[varindex] vprof.see_write(value) + @jit.unroll_safe def _all_locals_changed(self): for i, vprof in enumerate(self.getcode().vprofs): vprof.see_write(self.locals_cells_stack_w[i]) From noreply at buildbot.pypy.org Wed Aug 12 12:55:33 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 12:55:33 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (cfbolz, arigo aroundish): track precise classes too Message-ID: <20150812105533.200301C0726@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78930:1530921b69b5 Date: 2015-08-12 12:33 +0200 http://bitbucket.org/pypy/pypy/changeset/1530921b69b5/ Log: (cfbolz, arigo aroundish): track precise classes too diff --git a/pypy/interpreter/test/test_valueprof.py b/pypy/interpreter/test/test_valueprof.py --- a/pypy/interpreter/test/test_valueprof.py +++ b/pypy/interpreter/test/test_valueprof.py @@ -1,8 +1,10 @@ from pypy.interpreter.valueprof import * -class Value(): +class Value(object): pass +class OtherValue(Value): + pass class ValueInt(Value): def __init__(self, val): @@ -25,7 +27,7 @@ v.see_write(ValueInt(1)) v.see_write(ValueInt(1)) assert v.read_constant_int() == 1 - assert v._vprof_status == SEEN_INT + assert v._vprof_status == SEEN_CONSTANT_INT v.see_int(2) assert v._vprof_status == SEEN_TOO_MUCH v.see_int(1) @@ -53,7 +55,7 @@ v.see_write(value) v.see_write(value) assert v.try_read_constant_obj() is value - assert v._vprof_status == SEEN_OBJ + assert v._vprof_status == SEEN_CONSTANT_OBJ v.see_int(2) assert v._vprof_status == SEEN_TOO_MUCH @@ -74,12 +76,32 @@ v = ValueProf() v.see_write(ValueInt(1)) - assert v._vprof_status == SEEN_INT + assert v._vprof_status == SEEN_CONSTANT_INT v.see_write(None) assert v._vprof_status == SEEN_TOO_MUCH v = ValueProf() v.see_write(Value()) - assert v._vprof_status == SEEN_OBJ + assert v._vprof_status == SEEN_CONSTANT_OBJ v.see_write(None) assert v._vprof_status == SEEN_TOO_MUCH + +def test_known_class(): + v = ValueProf() + value = Value() + assert v._vprof_status == SEEN_NOTHING + v.see_write(value) + assert v._vprof_status == SEEN_CONSTANT_OBJ + v.see_write(Value()) + assert v._vprof_status == SEEN_CONSTANT_CLASS + v.see_write(OtherValue()) + assert v._vprof_status == SEEN_TOO_MUCH + + v = ValueProf() + assert v._vprof_status == SEEN_NOTHING + v.see_write(value) + assert v._vprof_status == SEEN_CONSTANT_OBJ + v.see_write(Value()) + assert v._vprof_status == SEEN_CONSTANT_CLASS + v.see_int(5) + assert v._vprof_status == SEEN_TOO_MUCH diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -1,10 +1,11 @@ from rpython.rlib import jit, objectmodel from rpython.rlib.rweakref import ref, dead_ref -SEEN_NOTHING = '\x00' -SEEN_INT = '\x01' -SEEN_OBJ = '\x02' -SEEN_TOO_MUCH = '\x03' +SEEN_NOTHING = '0' +SEEN_CONSTANT_INT = 'i' +SEEN_CONSTANT_OBJ = 'o' +SEEN_CONSTANT_CLASS = 'c' +SEEN_TOO_MUCH = '?' class ValueProf(object): _immutable_fields_ = ['_vprof_status?'] @@ -17,6 +18,7 @@ self._vprof_status = SEEN_NOTHING self._vprof_value_int = 0 self._vprof_value_wref = dead_ref + self._vprof_const_cls = None self._vprof_counter = 0 self._vprof_msg = msg @@ -38,8 +40,8 @@ status = self._vprof_status if status == SEEN_NOTHING: self._vprof_value_int = value - self._vprof_status = SEEN_INT - elif status == SEEN_INT: + self._vprof_status = SEEN_CONSTANT_INT + elif status == SEEN_CONSTANT_INT: if self.read_constant_int() != value: if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'int', value @@ -49,10 +51,12 @@ self._vprof_counter += 1 if self._vprof_counter == 200: print self._vprof_msg, 'int', value - elif status == SEEN_OBJ: + elif status == SEEN_CONSTANT_OBJ: self._vprof_status = SEEN_TOO_MUCH if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'int', value + elif status == SEEN_CONSTANT_CLASS: + self._vprof_status = SEEN_TOO_MUCH def see_object(self, value): status = self._vprof_status @@ -62,30 +66,41 @@ elif status == SEEN_NOTHING: try: self._vprof_value_wref = ref(value) - self._vprof_status = SEEN_OBJ + self._vprof_status = SEEN_CONSTANT_OBJ except TypeError: # for tests, which really use unwrapped ints in a few places self._vprof_status = SEEN_TOO_MUCH - elif status == SEEN_INT: + elif status == SEEN_CONSTANT_INT: if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value self._vprof_status = SEEN_TOO_MUCH - elif status == SEEN_OBJ: - if self.try_read_constant_obj() is not value: + elif status == SEEN_CONSTANT_OBJ: + prev_obj = self.try_read_constant_obj() + if prev_obj is not value: if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value - self._vprof_status = SEEN_TOO_MUCH + prev_cls = type(prev_obj) + if prev_cls is type(value): + self._vprof_const_cls = prev_cls + self._vprof_status = SEEN_CONSTANT_CLASS + else: + self._vprof_status = SEEN_TOO_MUCH else: if not jit.we_are_jitted(): self._vprof_counter += 1 if self._vprof_counter == 200: print self._vprof_msg, 'obj', value + elif status == SEEN_CONSTANT_CLASS: + cls = self.read_constant_cls() + if cls is not type(value): + self._vprof_status = SEEN_TOO_MUCH + def can_fold_read_int(self): - return self._vprof_status == SEEN_INT + return self._vprof_status == SEEN_CONSTANT_INT def can_fold_read_obj(self): - return self._vprof_status == SEEN_OBJ + return self._vprof_status == SEEN_CONSTANT_OBJ @jit.elidable def read_constant_int(self): @@ -97,4 +112,7 @@ assert self.can_fold_read_obj() return self._vprof_value_wref() + @jit.elidable + def read_constant_cls(self): + return self._vprof_const_cls From noreply at buildbot.pypy.org Wed Aug 12 12:55:35 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 12:55:35 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: track the type of integers too Message-ID: <20150812105535.54FF21C0726@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78931:0e8ef6a90f18 Date: 2015-08-12 12:55 +0200 http://bitbucket.org/pypy/pypy/changeset/0e8ef6a90f18/ Log: track the type of integers too diff --git a/pypy/interpreter/test/test_valueprof.py b/pypy/interpreter/test/test_valueprof.py --- a/pypy/interpreter/test/test_valueprof.py +++ b/pypy/interpreter/test/test_valueprof.py @@ -28,14 +28,18 @@ v.see_write(ValueInt(1)) assert v.read_constant_int() == 1 assert v._vprof_status == SEEN_CONSTANT_INT - v.see_int(2) - assert v._vprof_status == SEEN_TOO_MUCH - v.see_int(1) - assert v._vprof_status == SEEN_TOO_MUCH - v.see_int(2) - assert v._vprof_status == SEEN_TOO_MUCH - v.see_int(3) - assert v._vprof_status == SEEN_TOO_MUCH + v.see_write(ValueInt(2)) + assert v._vprof_status == SEEN_CONSTANT_CLASS + assert v._vprof_const_cls is ValueInt + v.see_write(ValueInt(1)) + assert v._vprof_status == SEEN_CONSTANT_CLASS + assert v._vprof_const_cls is ValueInt + v.see_write(ValueInt(2)) + assert v._vprof_status == SEEN_CONSTANT_CLASS + assert v._vprof_const_cls is ValueInt + v.see_write(ValueInt(3)) + assert v._vprof_status == SEEN_CONSTANT_CLASS + assert v._vprof_const_cls is ValueInt v = ValueProf() assert v._vprof_status == SEEN_NOTHING @@ -56,7 +60,7 @@ v.see_write(value) assert v.try_read_constant_obj() is value assert v._vprof_status == SEEN_CONSTANT_OBJ - v.see_int(2) + v.see_write(ValueInt(2)) assert v._vprof_status == SEEN_TOO_MUCH v = ValueProf() @@ -103,5 +107,5 @@ assert v._vprof_status == SEEN_CONSTANT_OBJ v.see_write(Value()) assert v._vprof_status == SEEN_CONSTANT_CLASS - v.see_int(5) + v.see_write(ValueInt(5)) assert v._vprof_status == SEEN_TOO_MUCH diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -33,10 +33,10 @@ if self._vprof_status == SEEN_TOO_MUCH: return if self.is_int(w_value): - return self.see_int(self.get_int_val(w_value)) - return self.see_object(w_value) + return self._see_int(self.get_int_val(w_value), w_value) + return self._see_object(w_value) - def see_int(self, value): + def _see_int(self, value, w_value): status = self._vprof_status if status == SEEN_NOTHING: self._vprof_value_int = value @@ -45,7 +45,8 @@ if self.read_constant_int() != value: if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'int', value - self._vprof_status = SEEN_TOO_MUCH + self._vprof_status = SEEN_CONSTANT_CLASS + self._vprof_const_cls = type(w_value) else: if not jit.we_are_jitted(): self._vprof_counter += 1 @@ -56,9 +57,10 @@ if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'int', value elif status == SEEN_CONSTANT_CLASS: - self._vprof_status = SEEN_TOO_MUCH + if type(w_value) is not self._vprof_const_cls: + self._vprof_status = SEEN_TOO_MUCH - def see_object(self, value): + def _see_object(self, value): status = self._vprof_status if value is None: if status != SEEN_TOO_MUCH: From noreply at buildbot.pypy.org Wed Aug 12 16:34:21 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 16:34:21 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: (fijal, arigo, plan_rich) fix test_dont_specialize_on_boxes_equal Message-ID: <20150812143421.66CD21C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78932:09e29a495996 Date: 2015-08-12 12:50 +0200 http://bitbucket.org/pypy/pypy/changeset/09e29a495996/ Log: (fijal, arigo, plan_rich) fix test_dont_specialize_on_boxes_equal diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -62,6 +62,20 @@ assert index >= 0 opinfo.setitem(index, self.res, pop, optheap=optheap) + def add_op_to_short(self, sb): + sop = self.getfield_op + preamble_arg = sb.produce_arg(sop.getarg(0)) + if preamble_arg is None: + return None + if sop.is_getfield(): + preamble_op = ResOperation(sop.getopnum(), [preamble_arg], + descr=sop.getdescr()) + else: + preamble_op = ResOperation(sop.getopnum(), [preamble_arg, + sop.getarg(1)], + descr=sop.getdescr()) + return ProducedShortOp(self, preamble_op) + def __repr__(self): return "HeapOp(%r)" % (self.res,) @@ -79,6 +93,17 @@ else: opt.pure(op.getopnum(), PreambleOp(op, preamble_op)) + def add_op_to_short(self, sb): + op = self.res + arglist = [] + for arg in op.getarglist(): + newarg = sb.produce_arg(arg) + if newarg is None: + return None + arglist.append(newarg) + return ProducedShortOp(self, op.copy_and_change(op.getopnum(), + args=arglist)) + def __repr__(self): return "PureOp(%r)" % (self.res,) @@ -94,9 +119,39 @@ key = make_hashable_int(op.getarg(0).getint()) optrewrite.loop_invariant_results[key] = PreambleOp(op, preamble_op) + def add_op_to_short(self, sb): + op = self.res + arglist = [] + for arg in op.getarglist(): + newarg = sb.produce_arg(arg) + if newarg is None: + return None + arglist.append(newarg) + opnum = OpHelpers.call_loopinvariant_for_descr(op.getdescr()) + return ProducedShortOp(self, op.copy_and_change(opnum, args=arglist)) + def __repr__(self): return "LoopInvariantOp(%r)" % (self.res,) +class CompoundOp(AbstractShortOp): + def __init__(self, res, one, two): + self.res = res + self.one = one + self.two = two + + def flatten(self, sb, l): + pop = self.one.add_op_to_short(sb) + if pop is not None: + l.append(pop) + two = self.two + if isinstance(two, CompoundOp): + two.flatten(sb, l) + else: + pop = two.add_op_to_short(sb) + if pop is not None: + l.append(pop) + return l + class AbstractProducedShortOp(object): pass @@ -111,6 +166,9 @@ def __repr__(self): return "%r -> %r" % (self.short_op, self.preamble_op) +dummy_short_op = ProducedShortOp(None, None) + + class ShortInputArg(AbstractProducedShortOp): def __init__(self, preamble_op): self.preamble_op = preamble_op @@ -131,6 +189,7 @@ # of AbstractShortOp self.potential_ops = {} self.produced_short_boxes = {} + self.extra_short_boxes = {} # a way to produce const boxes, e.g. setfield_gc(p0, Const). # We need to remember those, but they don't produce any new boxes self.const_short_boxes = [] @@ -151,8 +210,9 @@ self.add_op_to_short(shortop) # for op, produced_op in self.produced_short_boxes.iteritems(): - if isinstance(produced_op, ProducedShortOp): + if not isinstance(produced_op, ShortInputArg): short_boxes.append(produced_op) + short_boxes += self.extra_short_boxes for short_op in self.const_short_boxes: getfield_op = short_op.getfield_op @@ -173,7 +233,7 @@ elif isinstance(op, Const): return op elif op in self.potential_ops: - return self.add_op_to_short(self.potential_ops[op]) + return self.add_op_to_short(self.potential_ops[op]).preamble_op else: return None @@ -182,37 +242,26 @@ return # already added due to dependencies self.boxes_in_production[shortop.res] = None try: - op = shortop.res - if isinstance(shortop, HeapOp): - sop = shortop.getfield_op - preamble_arg = self.produce_arg(sop.getarg(0)) - if preamble_arg is None: - return None - if sop.is_getfield(): - preamble_op = ResOperation(sop.getopnum(), [preamble_arg], - descr=sop.getdescr()) + if isinstance(shortop, CompoundOp): + lst = shortop.flatten(self, []) + if len(lst) == 1: + pop = lst[0] else: - preamble_op = ResOperation(sop.getopnum(), [preamble_arg, - sop.getarg(1)], - descr=sop.getdescr()) + pop = lst[0] + for i in range(1, len(lst)): + opnum = OpHelpers.same_as_for_type(shortop.res.type) + new_name = ResOperation(opnum, [shortop.res]) + assert lst[i].short_op is not pop.short_op + lst[i].short_op.res = new_name + self.produced_short_boxes[new_name] = lst[i] else: - arglist = [] - for arg in op.getarglist(): - newarg = self.produce_arg(arg) - if newarg is None: - return None - arglist.append(newarg) - if isinstance(shortop, PureOp): - opnum = op.getopnum() - else: - opnum = OpHelpers.call_loopinvariant_for_descr( - op.getdescr()) - preamble_op = op.copy_and_change(opnum, args=arglist) - self.produced_short_boxes[op] = ProducedShortOp(shortop, - preamble_op) + pop = shortop.add_op_to_short(self) + if pop is None: + return + self.produced_short_boxes[shortop.res] = pop finally: del self.boxes_in_production[shortop.res] - return preamble_op + return pop def create_short_inputargs(self, label_args): short_inpargs = [] @@ -226,18 +275,26 @@ short_inpargs.append(inparg.preamble_op) return short_inpargs + def add_potential_op(self, op, pop): + prev_op = self.potential_ops.get(op, None) + if prev_op is None: + self.potential_ops[op] = pop + return + self.potential_ops[op] = CompoundOp(op, pop, prev_op) + def add_pure_op(self, op): - self.potential_ops[op] = PureOp(op) + assert op not in self.potential_ops + self.add_potential_op(op, PureOp(op)) def add_loopinvariant_op(self, op): - self.potential_ops[op] = LoopInvariantOp(op) + self.add_potential_op(op, LoopInvariantOp(op)) def add_heap_op(self, op, getfield_op): # or an inputarg if isinstance(op, Const) or op in self.produced_short_boxes: self.const_short_boxes.append(HeapOp(op, getfield_op)) return # we should not be called from anywhere - self.potential_ops[op] = HeapOp(op, getfield_op) + self.add_potential_op(op, HeapOp(op, getfield_op)) class EmptyInfo(info.AbstractInfo): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3678,7 +3678,6 @@ ''' expected = ''' [p1, i4, i3] - setfield_gc(p1, i3, descr=valuedescr) jump(p1, i3, i3) ''' preamble = ''' @@ -3704,7 +3703,6 @@ ''' expected = ''' [p1, i4, i3] - setfield_gc(p1, i3, descr=valuedescr) jump(p1, i3, i3) ''' preamble = ''' @@ -3747,7 +3745,7 @@ i3 = call_i(p1, descr=elidable3calldescr) guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) - i167 = same_as(i3) + i167 = same_as_i(i3) jump(p1, i4, i3) ''' self.optimize_loop(ops, expected, preamble) @@ -3770,7 +3768,7 @@ [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call_i(p1, descr=elidablecalldescr) - i151 = same_as(i3) + i151 = same_as_i(i3) jump(p1, i4, i3, i151) ''' self.optimize_loop(ops, expected, preamble) @@ -7388,7 +7386,7 @@ setfield_gc(p1, ii, descr=otherdescr) jump(p0, p1, ii2, ii, ii, ii) """ - self.optimize_loop(ops, expected) + self.optimize_loop(ops, expected, preamble) def test_dont_specialize_on_boxes_equal(self): ops = """ @@ -7396,7 +7394,7 @@ i1 = getfield_gc_i(p0, descr=valuedescr) i2 = getfield_gc_i(p1, descr=chardescr) setfield_gc(p3, i1, descr=adescr) - setfield_gc(p3, i2, descr=bdescr) + setfield_gc(p3, i2, descr=abisdescr) i4 = int_eq(i1, i2) guard_true(i4) [] i5 = int_gt(ii, 42) @@ -7406,7 +7404,7 @@ expected = """ [p0, p1, p3, ii, ii2, i1, i2] setfield_gc(p3, i1, descr=adescr) - setfield_gc(p3, i2, descr=bdescr) + setfield_gc(p3, i2, descr=abisdescr) i5 = int_gt(ii, 42) guard_true(i5) [] jump(p0, p1, p3, ii2, ii, i1, i2) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -172,9 +172,11 @@ floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float)) # a GcStruct not inheriting from OBJECT - S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('b', lltype.Ptr(NODE))) + S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed), + ('b', lltype.Ptr(NODE))) ssize = cpu.sizeof(S, False) adescr = cpu.fielddescrof(S, 'a') + abisdescr = cpu.fielddescrof(S, 'abis') bdescr = cpu.fielddescrof(S, 'b') #sbox = BoxPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S))) arraydescr2 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(S))) From noreply at buildbot.pypy.org Wed Aug 12 16:34:23 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 16:34:23 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix some tests Message-ID: <20150812143423.8781A1C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78933:b98fe23848c6 Date: 2015-08-12 16:24 +0200 http://bitbucket.org/pypy/pypy/changeset/b98fe23848c6/ Log: fix some tests diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -52,7 +52,8 @@ if unroll: opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) return opt.optimize_preamble(self.start_label, self.end_label, - self.operations) + self.operations, + self.call_pure_results) else: xxx @@ -93,7 +94,7 @@ assert unroll # we should not be here if it's disabled opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations) return opt.optimize_peeled_loop(self.start_label, self.end_jump, - self.operations, self.state) + self.operations, self.state, self.call_pure_results) def show_procedures(metainterp_sd, procedure=None, error=None): # debugging diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py --- a/rpython/jit/metainterp/optimizeopt/heap.py +++ b/rpython/jit/metainterp/optimizeopt/heap.py @@ -286,7 +286,10 @@ def clean_caches(self): del self._lazy_setfields_and_arrayitems[:] - for descr, cf in self.cached_fields.items(): + items = self.cached_fields.items() + if not we_are_translated(): + items.sort(key=str, reverse=True) + for descr, cf in items: if not descr.is_always_pure(): cf.invalidate(descr) for descr, submap in self.cached_arrayitems.iteritems(): @@ -474,8 +477,10 @@ cf.force_lazy_setfield(self, None, can_cache) def force_all_lazy_setfields_and_arrayitems(self): - # XXX fix the complexity here - for descr, cf in self.cached_fields.items(): + items = self.cached_fields.items() + if not we_are_translated(): + items.sort(key=str, reverse=True) + for descr, cf in items: cf.force_lazy_setfield(self, descr) for submap in self.cached_arrayitems.itervalues(): for index, cf in submap.iteritems(): diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -54,9 +54,11 @@ opinfo = opt.optimizer.ensure_ptr_info_arg0(g) pop = PreambleOp(self.res, preamble_op) assert not opinfo.is_virtual() + descr = self.getfield_op.getdescr() if g.is_getfield(): + cf = optheap.field_cache(descr) opinfo.setfield(preamble_op.getdescr(), self.res, pop, - optheap=optheap) + optheap, cf) else: index = g.getarg(1).getint() assert index >= 0 @@ -88,7 +90,7 @@ if optpure is None: return op = self.res - if op.is_call(): + if preamble_op.is_call(): optpure.extra_call_pure.append(PreambleOp(op, preamble_op)) else: opt.pure(op.getopnum(), PreambleOp(op, preamble_op)) @@ -101,8 +103,11 @@ if newarg is None: return None arglist.append(newarg) - return ProducedShortOp(self, op.copy_and_change(op.getopnum(), - args=arglist)) + if op.is_call(): + opnum = OpHelpers.call_pure_for_descr(op.getdescr()) + else: + opnum = op.getopnum() + return ProducedShortOp(self, op.copy_and_change(opnum, args=arglist)) def __repr__(self): return "PureOp(%r)" % (self.res,) @@ -189,7 +194,6 @@ # of AbstractShortOp self.potential_ops = {} self.produced_short_boxes = {} - self.extra_short_boxes = {} # a way to produce const boxes, e.g. setfield_gc(p0, Const). # We need to remember those, but they don't produce any new boxes self.const_short_boxes = [] @@ -212,7 +216,6 @@ for op, produced_op in self.produced_short_boxes.iteritems(): if not isinstance(produced_op, ShortInputArg): short_boxes.append(produced_op) - short_boxes += self.extra_short_boxes for short_op in self.const_short_boxes: getfield_op = short_op.getfield_op @@ -233,7 +236,10 @@ elif isinstance(op, Const): return op elif op in self.potential_ops: - return self.add_op_to_short(self.potential_ops[op]).preamble_op + r = self.add_op_to_short(self.potential_ops[op]) + if r is None: + return None + return r.preamble_op else: return None @@ -244,11 +250,20 @@ try: if isinstance(shortop, CompoundOp): lst = shortop.flatten(self, []) - if len(lst) == 1: - pop = lst[0] + if len(lst) == 0: + return None else: - pop = lst[0] - for i in range(1, len(lst)): + index = -1 + for i, item in enumerate(lst): + if not isinstance(item.short_op, HeapOp): + assert index == -1 + index = i + if index == -1: + index = 0 + pop = lst[index] + for i in range(len(lst)): + if i == index: + continue opnum = OpHelpers.same_as_for_type(shortop.res.type) new_name = ResOperation(opnum, [shortop.res]) assert lst[i].short_op is not pop.short_op diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3602,8 +3602,8 @@ [p1, i1, p2, i2] setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=readadescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) - setfield_gc(p2, i3, descr=adescr) jump(p1, i1, p2, i2) """ self.optimize_loop(ops, expected) @@ -3622,8 +3622,8 @@ [p1, i1, p2, i2] setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=writeadescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) - setfield_gc(p2, i3, descr=adescr) jump(p1, i1, p2, i2) """ self.optimize_loop(ops, expected) @@ -3634,8 +3634,8 @@ setfield_gc(p1, i1, descr=valuedescr) setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=plaincalldescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) - setfield_gc(p2, i3, descr=adescr) jump(p1, i1, p2, i2) """ self.optimize_loop(ops, ops) @@ -3678,6 +3678,7 @@ ''' expected = ''' [p1, i4, i3] + setfield_gc(p1, i3, descr=valuedescr) jump(p1, i3, i3) ''' preamble = ''' @@ -3703,6 +3704,7 @@ ''' expected = ''' [p1, i4, i3] + setfield_gc(p1, i3, descr=valuedescr) jump(p1, i3, i3) ''' preamble = ''' @@ -3760,16 +3762,16 @@ jump(p1, i4, i3) ''' expected = ''' - [p1, i4, i3, i5] + [p1, i4, i3] setfield_gc(p1, i4, descr=valuedescr) - jump(p1, i3, i5, i5) + jump(p1, i3, i3) ''' preamble = ''' [p1, i1, i4] setfield_gc(p1, i1, descr=valuedescr) i3 = call_i(p1, descr=elidablecalldescr) - i151 = same_as_i(i3) - jump(p1, i4, i3, i151) + # i151 = same_as_i(i3) + jump(p1, i4, i3) ''' self.optimize_loop(ops, expected, preamble) @@ -3789,36 +3791,36 @@ escape_n(i1) escape_n(i2) i4 = call_i(123456, 4, i0, 6, descr=elidablecalldescr) - i153 = same_as_i(i4) - jump(i0, i4, i153) + # i153 = same_as_i(i4) + jump(i0, i4) ''' expected = ''' - [i0, i4, i5] + [i0, i4] escape_n(42) escape_n(i4) - jump(i0, i5, i5) + jump(i0, i4) ''' self.optimize_loop(ops, expected, preamble, call_pure_results) def test_call_pure_constant_folding_memoryerr(self): ops = ''' [p0, i0] - escape(i0) - i3 = call_pure(123456, p0, descr=elidable2calldescr) + escape_n(i0) + i3 = call_pure_i(123456, p0, descr=elidable2calldescr) guard_no_exception() [] jump(p0, i3) ''' preamble = ''' [p0, i0] - escape(i0) - i3 = call(123456, p0, descr=elidable2calldescr) + escape_n(i0) + i3 = call_i(123456, p0, descr=elidable2calldescr) guard_no_exception() [] - i4 = same_as(i3) + # i4 = same_as(i3) jump(p0, i3, i4) ''' expected = ''' [p0, i3, i4] - escape(i3) + escape_n(i3) jump(p0, i4, i4) ''' self.optimize_loop(ops, expected, preamble) @@ -8333,18 +8335,18 @@ def test_cache_setarrayitem_across_loop_boundaries(self): ops = """ [p1] - p2 = getarrayitem_gc_r(p1, 3, descr=arraydescr) + p2 = getarrayitem_gc_r(p1, 3, descr=gcarraydescr) guard_nonnull_class(p2, ConstClass(node_vtable)) [] call_n(p2, descr=nonwritedescr) p3 = new_with_vtable(descr=nodesize) - setarrayitem_gc(p1, 3, p3, descr=arraydescr) + setarrayitem_gc(p1, 3, p3, descr=gcarraydescr) jump(p1) """ expected = """ [p1, p2] call_n(p2, descr=nonwritedescr) p3 = new_with_vtable(descr=nodesize) - setarrayitem_gc(p1, 3, p3, descr=arraydescr) + setarrayitem_gc(p1, 3, p3, descr=gcarraydescr) jump(p1, p3) """ self.optimize_loop(ops, expected) @@ -8392,8 +8394,8 @@ [i0, i1] p0 = escape_r() p1 = escape_r() + setfield_gc(p1, p0, descr=bdescr) setfield_gc(p0, p1, descr=adescr) - setfield_gc(p1, p0, descr=bdescr) jump(i0, i1) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -20,6 +20,7 @@ from rpython.jit.metainterp.counter import DeterministicJitCounter from rpython.config.translationoption import get_combined_translation_config from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgRef +from rpython.jit.metainterp.optimizeopt.util import args_dict def test_sort_descrs(): @@ -438,6 +439,12 @@ metainterp_sd.callinfocollection = self.callinfocollection # compile_data.enable_opts = self.enable_opts + new_call_pure_results = args_dict() + if call_pure_results is not None: + for k, v in call_pure_results.items(): + new_call_pure_results[list(k)] = v + + compile_data.call_pure_results = new_call_pure_results state = optimize_trace(metainterp_sd, None, compile_data) return state diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -37,6 +37,8 @@ def setinfo_from_preamble(self, op, preamble_info, exported_infos): op = self.get_box_replacement(op) + if op.get_forwarded() is not None: + return # XXX? if isinstance(preamble_info, info.PtrInfo): if preamble_info.is_virtual(): # XXX do we want to sanitize this? @@ -88,10 +90,10 @@ assert op.get_forwarded() is None assert not self.optimizer._newoperations - def optimize_preamble(self, start_label, end_label, ops): + def optimize_preamble(self, start_label, end_label, ops, call_pure_results): self._check_no_forwarding([[start_label, end_label], ops]) info, newops = self.optimizer.propagate_all_forward( - start_label.getarglist()[:], ops) + start_label.getarglist()[:], ops, call_pure_results) exported_state = self.export_state(start_label, end_label, info.inputargs) # we need to absolutely make sure that we've cleaned up all @@ -99,12 +101,13 @@ self.optimizer._clean_optimization_info(self.optimizer._newoperations) return exported_state, self.optimizer._newoperations - def optimize_peeled_loop(self, start_label, end_jump, ops, state): + def optimize_peeled_loop(self, start_label, end_jump, ops, state, + call_pure_results): self._check_no_forwarding([[start_label, end_jump], ops]) self.import_state(start_label, state) self.potential_extra_ops = {} self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops, - rename_inputargs=False) + call_pure_results, False) orig_jump_args = [self.get_box_replacement(op) for op in end_jump.getarglist()] jump_args = state.virtual_state.make_inputargs(orig_jump_args, From noreply at buildbot.pypy.org Wed Aug 12 16:34:25 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 16:34:25 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix again test_varray_boxed1 Message-ID: <20150812143425.9EB2C1C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78934:18012a6f5352 Date: 2015-08-12 16:34 +0200 http://bitbucket.org/pypy/pypy/changeset/18012a6f5352/ Log: fix again test_varray_boxed1 diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -45,12 +45,16 @@ self.res = res self.getfield_op = getfield_op - def produce_op(self, opt, preamble_op): + def produce_op(self, opt, preamble_op, exported_infos): optheap = opt.optimizer.optheap if optheap is None: return g = preamble_op.copy_and_change(preamble_op.getopnum(), args=self.getfield_op.getarglist()) + if g.getarg(0) in exported_infos: + opt.optimizer.setinfo_from_preamble(g.getarg(0), + exported_infos[g.getarg(0)], + exported_infos) opinfo = opt.optimizer.ensure_ptr_info_arg0(g) pop = PreambleOp(self.res, preamble_op) assert not opinfo.is_virtual() @@ -85,7 +89,7 @@ def __init__(self, res): self.res = res - def produce_op(self, opt, preamble_op): + def produce_op(self, opt, preamble_op, exported_infos): optpure = opt.optimizer.optpure if optpure is None: return @@ -116,7 +120,7 @@ def __init__(self, res): self.res = res - def produce_op(self, opt, preamble_op): + def produce_op(self, opt, preamble_op, exported_infos): optrewrite = opt.optimizer.optrewrite if optrewrite is None: return @@ -165,8 +169,8 @@ self.short_op = short_op self.preamble_op = preamble_op - def produce_op(self, opt): - self.short_op.produce_op(opt, self.preamble_op) + def produce_op(self, opt, exported_infos): + self.short_op.produce_op(opt, self.preamble_op, exported_infos) def __repr__(self): return "%r -> %r" % (self.short_op, self.preamble_op) @@ -178,7 +182,7 @@ def __init__(self, preamble_op): self.preamble_op = preamble_op - def produce_op(self, opt): + def produce_op(self, opt, exported_infos): pass def __repr__(self): diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -38,7 +38,7 @@ def setinfo_from_preamble(self, op, preamble_info, exported_infos): op = self.get_box_replacement(op) if op.get_forwarded() is not None: - return # XXX? + return if isinstance(preamble_info, info.PtrInfo): if preamble_info.is_virtual(): # XXX do we want to sanitize this? @@ -338,7 +338,7 @@ exported_state.short_boxes, exported_state.short_inputargs, exported_state.exported_infos, self.optimizer) for produced_op in exported_state.short_boxes: - produced_op.produce_op(self) + produced_op.produce_op(self, exported_state.exported_infos) return self.inputargs = targetop.getarglist() From noreply at buildbot.pypy.org Wed Aug 12 17:57:54 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 17:57:54 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: (cfbolz, arigo around): refactor Message-ID: <20150812155754.74AE21C11C8@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78935:93b464bf7515 Date: 2015-08-12 13:08 +0200 http://bitbucket.org/pypy/pypy/changeset/93b464bf7515/ Log: (cfbolz, arigo around): refactor diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -30,59 +30,48 @@ @objectmodel.always_inline def see_write(self, w_value): - if self._vprof_status == SEEN_TOO_MUCH: + status = self._vprof_status + if status == SEEN_TOO_MUCH: return - if self.is_int(w_value): - return self._see_int(self.get_int_val(w_value), w_value) - return self._see_object(w_value) - def _see_int(self, value, w_value): - status = self._vprof_status + if w_value is None: + self._vprof_status = SEEN_TOO_MUCH + return + if status == SEEN_NOTHING: - self._vprof_value_int = value - self._vprof_status = SEEN_CONSTANT_INT + if self.is_int(w_value): + self._vprof_value_int = self.get_int_val(w_value) + self._vprof_status = SEEN_CONSTANT_INT + else: + try: + self._vprof_value_wref = ref(w_value) + self._vprof_status = SEEN_CONSTANT_OBJ + except TypeError: + # for tests, which really use unwrapped ints in a few places + self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_CONSTANT_INT: - if self.read_constant_int() != value: + if self.is_int(w_value): + if self.read_constant_int() != self.get_int_val(w_value): + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT", self._vprof_msg, 'int', w_value + self._vprof_status = SEEN_CONSTANT_CLASS + self._vprof_const_cls = type(w_value) + else: + if not jit.we_are_jitted(): + self._vprof_counter += 1 + if self._vprof_counter == 200: + print self._vprof_msg, 'int', w_value + else: if self._vprof_counter >= 200: - print "NO LONGER CONSTANT", self._vprof_msg, 'int', value - self._vprof_status = SEEN_CONSTANT_CLASS - self._vprof_const_cls = type(w_value) - else: - if not jit.we_are_jitted(): - self._vprof_counter += 1 - if self._vprof_counter == 200: - print self._vprof_msg, 'int', value - elif status == SEEN_CONSTANT_OBJ: - self._vprof_status = SEEN_TOO_MUCH - if self._vprof_counter >= 200: - print "NO LONGER CONSTANT", self._vprof_msg, 'int', value - elif status == SEEN_CONSTANT_CLASS: - if type(w_value) is not self._vprof_const_cls: + print "NO LONGER CONSTANT", self._vprof_msg, 'obj', w_value self._vprof_status = SEEN_TOO_MUCH - - def _see_object(self, value): - status = self._vprof_status - if value is None: - if status != SEEN_TOO_MUCH: - self._vprof_status = SEEN_TOO_MUCH - elif status == SEEN_NOTHING: - try: - self._vprof_value_wref = ref(value) - self._vprof_status = SEEN_CONSTANT_OBJ - except TypeError: - # for tests, which really use unwrapped ints in a few places - self._vprof_status = SEEN_TOO_MUCH - elif status == SEEN_CONSTANT_INT: - if self._vprof_counter >= 200: - print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value - self._vprof_status = SEEN_TOO_MUCH elif status == SEEN_CONSTANT_OBJ: prev_obj = self.try_read_constant_obj() - if prev_obj is not value: + if prev_obj is not w_value: if self._vprof_counter >= 200: - print "NO LONGER CONSTANT", self._vprof_msg, 'obj', value + print "NO LONGER CONSTANT", self._vprof_msg, 'obj', w_value prev_cls = type(prev_obj) - if prev_cls is type(value): + if prev_cls is type(w_value): self._vprof_const_cls = prev_cls self._vprof_status = SEEN_CONSTANT_CLASS else: @@ -91,13 +80,12 @@ if not jit.we_are_jitted(): self._vprof_counter += 1 if self._vprof_counter == 200: - print self._vprof_msg, 'obj', value + print self._vprof_msg, 'obj', w_value elif status == SEEN_CONSTANT_CLASS: cls = self.read_constant_cls() - if cls is not type(value): + if cls is not type(w_value): self._vprof_status = SEEN_TOO_MUCH - def can_fold_read_int(self): return self._vprof_status == SEEN_CONSTANT_INT From noreply at buildbot.pypy.org Wed Aug 12 17:57:56 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 17:57:56 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: print about classes too Message-ID: <20150812155756.839DE1C11C8@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78936:7bb77134dbae Date: 2015-08-12 13:15 +0200 http://bitbucket.org/pypy/pypy/changeset/7bb77134dbae/ Log: print about classes too diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -52,8 +52,9 @@ elif status == SEEN_CONSTANT_INT: if self.is_int(w_value): if self.read_constant_int() != self.get_int_val(w_value): - if self._vprof_counter >= 200: - print "NO LONGER CONSTANT", self._vprof_msg, 'int', w_value + if not jit.we_are_jitted(): + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT", self._vprof_msg, 'int', w_value self._vprof_status = SEEN_CONSTANT_CLASS self._vprof_const_cls = type(w_value) else: @@ -85,6 +86,13 @@ cls = self.read_constant_cls() if cls is not type(w_value): self._vprof_status = SEEN_TOO_MUCH + if self._vprof_counter >= 200: + print "NO LONGER CONSTANT CLASS", self._vprof_msg, 'cls', cls + else: + if not jit.we_are_jitted(): + self._vprof_counter += 1 + if self._vprof_counter == 200: + print self._vprof_msg, 'cls', cls def can_fold_read_int(self): return self._vprof_status == SEEN_CONSTANT_INT @@ -92,6 +100,9 @@ def can_fold_read_obj(self): return self._vprof_status == SEEN_CONSTANT_OBJ + def class_is_known(self): + return self._vprof_status == SEEN_CONSTANT_CLASS + @jit.elidable def read_constant_int(self): assert self.can_fold_read_int() From noreply at buildbot.pypy.org Wed Aug 12 17:57:58 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 17:57:58 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: feed the known classes to the JIT Message-ID: <20150812155758.969881C11C8@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78937:0fbb6b5d0ff3 Date: 2015-08-12 17:45 +0200 http://bitbucket.org/pypy/pypy/changeset/0fbb6b5d0ff3/ Log: feed the known classes to the JIT diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -147,9 +147,9 @@ def _getlocal(self, varindex): from pypy.objspace.std.intobject import W_IntObject + # some careful logic there if we_are_jitted(): vprof = self.getcode().vprofs[varindex] - # some careful logic there if vprof.can_fold_read_int(): return W_IntObject(vprof.read_constant_int()) elif vprof.can_fold_read_obj(): @@ -157,6 +157,8 @@ if w_res is not None: return w_res w_res = self.locals_cells_stack_w[varindex] + if we_are_jitted() and vprof.class_is_known(): + jit.record_known_class(w_res, vprof.read_constant_cls()) return w_res def _setlocal(self, varindex, value): diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -36,12 +36,13 @@ if attr is None: return self.terminator._read_terminator(obj, selector) # XXX move to PlainAttribute? - if attr.can_fold_read_int(): - return W_IntObject(attr.read_constant_int()) - elif attr.can_fold_read_obj(): - w_res = attr.try_read_constant_obj() - if w_res is not None: - return w_res + if jit.we_are_jitted(): + if attr.can_fold_read_int(): + return W_IntObject(attr.read_constant_int()) + elif attr.can_fold_read_obj(): + w_res = attr.try_read_constant_obj() + if w_res is not None: + return w_res if ( jit.isconstant(attr.storageindex) and jit.isconstant(obj) and @@ -49,7 +50,10 @@ ): return self._pure_mapdict_read_storage(obj, attr.storageindex) else: - return obj._mapdict_read_storage(attr.storageindex) + w_res = obj._mapdict_read_storage(attr.storageindex) + if jit.we_are_jitted() and attr.class_is_known(): + jit.record_known_class(w_res, attr.read_constant_cls()) + return w_res @jit.elidable def _pure_mapdict_read_storage(self, obj, storageindex): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -350,6 +350,7 @@ assert c.terminator.size_estimate() in [(i + 10) // 2, (i + 11) // 2] def test_value_profiling(monkeypatch): + monkeypatch.setattr(jit, "we_are_jitted", lambda : True) class Value: pass a = Value() @@ -374,6 +375,28 @@ assert not obj.map.can_fold_read_obj() +def test_value_profiling_known_cls(monkeypatch): + class Value(object): + pass + a = Value() + cls = Class() + obj = cls.instantiate() + a1 = Value() + obj.setdictvalue(space, "a", a1) + obj = cls.instantiate() + obj.setdictvalue(space, "a", a1) + obj.setdictvalue(space, "a", a) + + seen = [] + def f(obj, cls): + seen.append((obj, cls)) + monkeypatch.setattr(jit, "we_are_jitted", lambda : True) + monkeypatch.setattr(jit, "record_known_class", f) + + assert obj.getdictvalue(space, "a") == a + assert obj.getdictvalue(space, "a") == a + assert seen == [(a, Value), (a, Value)] + # ___________________________________________________________ # dict tests From noreply at buildbot.pypy.org Wed Aug 12 17:58:00 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 17:58:00 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: maybe fix translation Message-ID: <20150812155800.9CB531C11C8@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78938:4f22acb2175c Date: 2015-08-12 17:58 +0200 http://bitbucket.org/pypy/pypy/changeset/4f22acb2175c/ Log: maybe fix translation diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py --- a/pypy/interpreter/valueprof.py +++ b/pypy/interpreter/valueprof.py @@ -56,7 +56,7 @@ if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'int', w_value self._vprof_status = SEEN_CONSTANT_CLASS - self._vprof_const_cls = type(w_value) + self._vprof_const_cls = w_value.__class__ else: if not jit.we_are_jitted(): self._vprof_counter += 1 @@ -71,8 +71,8 @@ if prev_obj is not w_value: if self._vprof_counter >= 200: print "NO LONGER CONSTANT", self._vprof_msg, 'obj', w_value - prev_cls = type(prev_obj) - if prev_cls is type(w_value): + prev_cls = prev_obj.__class__ + if prev_cls is w_value.__class__: self._vprof_const_cls = prev_cls self._vprof_status = SEEN_CONSTANT_CLASS else: @@ -84,7 +84,7 @@ print self._vprof_msg, 'obj', w_value elif status == SEEN_CONSTANT_CLASS: cls = self.read_constant_cls() - if cls is not type(w_value): + if cls is not w_value.__class__: self._vprof_status = SEEN_TOO_MUCH if self._vprof_counter >= 200: print "NO LONGER CONSTANT CLASS", self._vprof_msg, 'cls', cls From noreply at buildbot.pypy.org Wed Aug 12 18:05:19 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 18:05:19 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: another translation problem Message-ID: <20150812160519.E4C2E1C133C@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78939:6ef217777d4d Date: 2015-08-12 18:05 +0200 http://bitbucket.org/pypy/pypy/changeset/6ef217777d4d/ Log: another translation problem diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -148,6 +148,7 @@ def _getlocal(self, varindex): from pypy.objspace.std.intobject import W_IntObject # some careful logic there + vprof = None if we_are_jitted(): vprof = self.getcode().vprofs[varindex] if vprof.can_fold_read_int(): From noreply at buildbot.pypy.org Wed Aug 12 18:56:01 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:01 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix (again) issue 2015 Message-ID: <20150812165601.11CCD1C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78940:a858bb9a544c Date: 2015-08-12 16:41 +0200 http://bitbucket.org/pypy/pypy/changeset/a858bb9a544c/ Log: fix (again) issue 2015 diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -157,17 +157,13 @@ opnum = OpHelpers.call_for_descr(op.getdescr()) newop = self.optimizer.replace_op_with(op, opnum) self.emit_operation(newop) - # don't move call_pure_with_exception in the short preamble... - # issue #2015 + # don't move call_pure_with_exception in the short preamble... + # issue #2015 - # XXX default has this code: - # this does not work with how pure calls are done on this branch - # fix together with unroll - #effectinfo = op.getdescr().get_extra_info() - #if not effectinfo.check_can_raise(ignore_memoryerror=True): - # if 1: - self.call_pure_positions.append( - len(self.optimizer._newoperations) - 1) + effectinfo = op.getdescr().get_extra_info() + if not effectinfo.check_can_raise(ignore_memoryerror=True): + self.call_pure_positions.append( + len(self.optimizer._newoperations) - 1) optimize_CALL_PURE_R = optimize_CALL_PURE_I optimize_CALL_PURE_F = optimize_CALL_PURE_I diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3631,8 +3631,8 @@ def test_residual_call_invalidates_some_read_caches_3(self): ops = """ [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) setfield_gc(p1, i1, descr=valuedescr) - setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=plaincalldescr) setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) @@ -3747,7 +3747,6 @@ i3 = call_i(p1, descr=elidable3calldescr) guard_no_exception() [] setfield_gc(p1, i3, descr=valuedescr) - i167 = same_as_i(i3) jump(p1, i4, i3) ''' self.optimize_loop(ops, expected, preamble) From noreply at buildbot.pypy.org Wed Aug 12 18:56:03 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:03 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: (arigo, fijal) copy contents of heap cache when stuff is proven constant Message-ID: <20150812165603.31E401C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78941:49c5c24ec474 Date: 2015-08-12 17:42 +0200 http://bitbucket.org/pypy/pypy/changeset/49c5c24ec474/ Log: (arigo, fijal) copy contents of heap cache when stuff is proven constant diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -51,6 +51,9 @@ def getstrlen(self, op, string_optimizer, mode, create_ops=True): return None + def copy_fields_to_const(self, constinfo, optheap): + pass + def make_guards(self, op, short): pass @@ -128,6 +131,11 @@ assert not self.is_virtual() self._fields = [None] * len(self._fields) + def copy_fields_to_const(self, constinfo, optheap): + if self._fields is not None: + info = constinfo._get_info(optheap) + info._fields = self._fields[:] + def all_items(self): return self._fields @@ -472,12 +480,11 @@ def getconst(self): return self._const - def _get_info(self, descr, optheap): + def _get_info(self, optheap): ref = self._const.getref_base() info = optheap.const_infos.get(ref, None) if info is None: info = StructPtrInfo() - info.init_fields(descr.get_parent_descr(), descr.get_index()) optheap.const_infos[ref] = info return info @@ -490,7 +497,7 @@ return info def getfield(self, descr, optheap=None): - info = self._get_info(descr, optheap) + info = self._get_info(optheap) return info.getfield(descr) def getitem(self, index, optheap=None): @@ -502,7 +509,7 @@ info.setitem(index, op, cf) def setfield(self, descr, struct, op, optheap=None, cf=None): - info = self._get_info(descr, optheap) + info = self._get_info(optheap) info.setfield(descr, struct, op, optheap, cf) def is_null(self): diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -387,6 +387,9 @@ assert box.get_forwarded().contains(constbox.getint()) if box.is_constant(): return + if box.type == 'r' and box.get_forwarded() is not None: + opinfo = box.get_forwarded() + opinfo.copy_fields_to_const(self.getptrinfo(constbox), self.optheap) box.set_forwarded(constbox) def make_constant_int(self, box, intvalue): diff --git a/rpython/jit/metainterp/optimizeopt/pure.py b/rpython/jit/metainterp/optimizeopt/pure.py --- a/rpython/jit/metainterp/optimizeopt/pure.py +++ b/rpython/jit/metainterp/optimizeopt/pure.py @@ -157,13 +157,8 @@ opnum = OpHelpers.call_for_descr(op.getdescr()) newop = self.optimizer.replace_op_with(op, opnum) self.emit_operation(newop) - # don't move call_pure_with_exception in the short preamble... - # issue #2015 - - effectinfo = op.getdescr().get_extra_info() - if not effectinfo.check_can_raise(ignore_memoryerror=True): - self.call_pure_positions.append( - len(self.optimizer._newoperations) - 1) + self.call_pure_positions.append( + len(self.optimizer._newoperations) - 1) optimize_CALL_PURE_R = optimize_CALL_PURE_I optimize_CALL_PURE_F = optimize_CALL_PURE_I @@ -229,8 +224,13 @@ sb.add_pure_op(op) for i in self.call_pure_positions: op = ops[i] - assert op.is_call() - sb.add_pure_op(op) + # don't move call_pure_with_exception in the short preamble... + # issue #2015 + + effectinfo = op.getdescr().get_extra_info() + if not effectinfo.check_can_raise(ignore_memoryerror=True): + assert op.is_call() + sb.add_pure_op(op) dispatch_opt = make_dispatcher_method(OptPure, 'optimize_', default=OptPure.optimize_default) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3814,13 +3814,12 @@ escape_n(i0) i3 = call_i(123456, p0, descr=elidable2calldescr) guard_no_exception() [] - # i4 = same_as(i3) - jump(p0, i3, i4) + jump(p0, i3) ''' expected = ''' - [p0, i3, i4] + [p0, i3] escape_n(i3) - jump(p0, i4, i4) + jump(p0, i3) ''' self.optimize_loop(ops, expected, preamble) @@ -3850,7 +3849,7 @@ escape_n(i1) escape_n(i2) escape_n(i9) - i4 = call(123456, 4, i0, 6, descr=elidable3calldescr) + i4 = call_i(123456, 4, i0, 6, descr=elidable3calldescr) guard_no_exception() [] jump(i0, i4) ''' @@ -3858,8 +3857,8 @@ [i0, i2] escape_n(42) escape_n(i2) - escape(i2) - i4 = call(123456, 4, i0, 6, descr=elidable3calldescr) + escape_n(i2) + i4 = call_i(123456, 4, i0, 6, descr=elidable3calldescr) guard_no_exception() [] jump(i0, i4) ''' From noreply at buildbot.pypy.org Wed Aug 12 18:56:05 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:05 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix the test Message-ID: <20150812165605.48A041C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78942:76446f3fe321 Date: 2015-08-12 17:46 +0200 http://bitbucket.org/pypy/pypy/changeset/76446f3fe321/ Log: fix the test diff --git a/rpython/jit/metainterp/optimizeopt/TODO b/rpython/jit/metainterp/optimizeopt/TODO --- a/rpython/jit/metainterp/optimizeopt/TODO +++ b/rpython/jit/metainterp/optimizeopt/TODO @@ -1,6 +1,4 @@ -* arraylen_gc is not handling length bound optimization at all (we need to - wait till unrolling for tests) * mark_opaque_pointer is ignored (which is fine until unrolling) -* clean up and unify _corresponding_vtable and vtable fields on SizeDescr - and check if all the usecases are necessary * implement unrolling +* implement more cases of copying _fields and _items between normal info + and ConstPtrInfo when proven constant diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py --- a/rpython/jit/metainterp/optimizeopt/info.py +++ b/rpython/jit/metainterp/optimizeopt/info.py @@ -344,6 +344,11 @@ def all_items(self): return self._items + def copy_fields_to_const(self, constinfo, optheap): + if self._items is not None: + info = constinfo._get_array_info(optheap) + info._items = self._items[:] + def _force_elements(self, op, optforce, descr): arraydescr = op.getdescr() for i in range(self.length): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -6004,22 +6004,20 @@ def test_let_getfield_kill_setfields(self): ops = """ [p0] - p1 = getfield_gc_r(p0, descr=valuedescr) - setfield_gc(p0, p1, descr=valuedescr) - setfield_gc(p0, p1, descr=valuedescr) - setfield_gc(p0, p0, descr=valuedescr) + p1 = getfield_gc_r(p0, descr=nextdescr) + setfield_gc(p0, p1, descr=nextdescr) + setfield_gc(p0, p1, descr=nextdescr) + setfield_gc(p0, p0, descr=nextdescr) jump(p0) """ preamble = """ [p0] - p1 = getfield_gc_r(p0, descr=valuedescr) - setfield_gc(p0, p0, descr=valuedescr) - p4450 = same_as(p0) # Should be killed by backend + p1 = getfield_gc_r(p0, descr=nextdescr) + setfield_gc(p0, p0, descr=nextdescr) jump(p0) """ expected = """ [p0] - setfield_gc(p0, p0, descr=valuedescr) jump(p0) """ self.optimize_loop(ops, expected, preamble) From noreply at buildbot.pypy.org Wed Aug 12 18:56:07 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:07 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: (arigo, fijal) fix const handling in the short preamble Message-ID: <20150812165607.4F2B71C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78943:57b9a085d148 Date: 2015-08-12 17:56 +0200 http://bitbucket.org/pypy/pypy/changeset/57b9a085d148/ Log: (arigo, fijal) fix const handling in the short preamble diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py --- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py +++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py @@ -323,14 +323,12 @@ class ShortPreambleBuilder(object): def __init__(self, short_boxes, short_inputargs, exported_infos, optimizer=None): - self.producable_ops = {} for produced_op in short_boxes: op = produced_op.short_op.res preamble_op = produced_op.preamble_op if isinstance(op, Const): info = optimizer.getinfo(op) else: - self.producable_ops[op] = produced_op.preamble_op info = exported_infos[op] if info is None: info = empty_info @@ -340,11 +338,7 @@ self.short_preamble_jump = [] self.short_inputargs = short_inputargs - def use_box(self, box, optimizer=None): - preamble_op = self.producable_ops.get(box, None) - if preamble_op is None: - return - del self.producable_ops[box] + def use_box(self, box, preamble_op, optimizer=None): for arg in preamble_op.getarglist(): if isinstance(arg, Const): pass diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7912,12 +7912,18 @@ setfield_gc(p5, 1, descr=valuedescr) jump(p5) """ + short = """ + [p0] + i0 = getfield_gc_i(p0, descr=valuedescr) + guard_value(i0, 1) [] + jump() + """ expected = """ [p5] call_n(1, descr=nonwritedescr) jump(p5) """ - self.optimize_loop(ops, expected, preamble) + self.optimize_loop(ops, expected, preamble, expected_short=short) def test_dont_mixup_equal_boxes(self): ops = """ diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -22,8 +22,11 @@ if isinstance(preamble_op, PreambleOp): op = preamble_op.op self.optimizer.inparg_dict[op] = None # XXX ARGH - self.optunroll.short_preamble_producer.use_box(op, self) - self.optunroll.potential_extra_ops[op] = preamble_op + # special hack for int_add(x, accumulator-const) optimization + self.optunroll.short_preamble_producer.use_box(op, + preamble_op.preamble_op, self) + if not preamble_op.op.is_constant(): + self.optunroll.potential_extra_ops[op] = preamble_op return op return preamble_op From noreply at buildbot.pypy.org Wed Aug 12 18:56:09 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:09 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: (fijal, arigo) fix more tests Message-ID: <20150812165609.5AAA11C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78944:bedcdcb6f10f Date: 2015-08-12 18:11 +0200 http://bitbucket.org/pypy/pypy/changeset/bedcdcb6f10f/ Log: (fijal, arigo) fix more tests diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -420,6 +420,7 @@ resvalue = self.loop_invariant_results.get(key, None) if resvalue is not None: resvalue = self.optimizer.force_op_from_preamble(resvalue) + self.loop_invariant_results[key] = resvalue self.make_equal_to(op, resvalue) self.last_emitted_operation = REMOVED return diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3953,20 +3953,20 @@ p3 = force_token() # p2 = new_with_vtable(descr=vref_descr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p2, NULL, descr=virtualforceddescr) - setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force_n(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] # setfield_gc(p0, NULL, descr=nextdescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) p1 = new_with_vtable(descr=nodesize) p1b = new_with_vtable(descr=nodesize) setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ self.optimize_loop(ops, expected, expected) @@ -3993,20 +3993,20 @@ p3 = force_token() # p2 = new_with_vtable(descr=vref_descr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p2, NULL, descr=virtualforceddescr) - setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force_n(i1, descr=mayforcevirtdescr) guard_not_forced() [p2, i1] # setfield_gc(p0, NULL, descr=nextdescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) p1 = new_with_vtable(descr=nodesize) p1b = new_with_vtable(descr=nodesize) setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args @@ -4081,8 +4081,8 @@ [i1] p3 = force_token() p2 = new_with_vtable(descr=vref_descr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p2, NULL, descr=virtualforceddescr) - setfield_gc(p2, p3, descr=virtualtokendescr) escape_n(p2) p1 = new_with_vtable(descr=nodesize) setfield_gc(p2, p1, descr=virtualforceddescr) @@ -4106,9 +4106,9 @@ expected = """ [i1, p1] p3 = force_token() - p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) + p2 = new_with_vtable(descr=vref_descr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p2, NULL, descr=virtualforceddescr) - setfield_gc(p2, p3, descr=virtualtokendescr) escape_n(p2) setfield_gc(p2, p1, descr=virtualforceddescr) setfield_gc(p2, NULL, descr=virtualtokendescr) @@ -4888,9 +4888,9 @@ ops = """ [p4, p7, i30] p16 = getfield_gc_r(p4, descr=valuedescr) - p17 = getarrayitem_gc_r(p4, 1, descr=arraydescr) + p17 = getarrayitem_gc_r(p4, 1, descr=gcarraydescr) guard_value(p16, ConstPtr(myptr), descr=) [] - i1 = getfield_raw(p7, descr=nextdescr) + i1 = getfield_raw_i(p7, descr=nextdescr) i2 = int_add(i1, i30) setfield_raw(p7, 7, descr=nextdescr) setfield_raw(p7, i2, descr=nextdescr) @@ -4898,7 +4898,7 @@ """ expected = """ [p4, p7, i30] - i1 = getfield_raw(p7, descr=nextdescr) + i1 = getfield_raw_i(p7, descr=nextdescr) i2 = int_add(i1, i30) setfield_raw(p7, 7, descr=nextdescr) setfield_raw(p7, i2, descr=nextdescr) diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -24,7 +24,7 @@ self.optimizer.inparg_dict[op] = None # XXX ARGH # special hack for int_add(x, accumulator-const) optimization self.optunroll.short_preamble_producer.use_box(op, - preamble_op.preamble_op, self) + preamble_op.preamble_op, self) if not preamble_op.op.is_constant(): self.optunroll.potential_extra_ops[op] = preamble_op return op From noreply at buildbot.pypy.org Wed Aug 12 18:56:11 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:11 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: (fijal, arigo) fix the test Message-ID: <20150812165611.619C71C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78945:b5a725af47cc Date: 2015-08-12 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/b5a725af47cc/ Log: (fijal, arigo) fix the test diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4888,7 +4888,6 @@ ops = """ [p4, p7, i30] p16 = getfield_gc_r(p4, descr=valuedescr) - p17 = getarrayitem_gc_r(p4, 1, descr=gcarraydescr) guard_value(p16, ConstPtr(myptr), descr=) [] i1 = getfield_raw_i(p7, descr=nextdescr) i2 = int_add(i1, i30) From noreply at buildbot.pypy.org Wed Aug 12 18:56:13 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:13 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix some tests Message-ID: <20150812165613.6CB661C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78946:3371b0e4ea3a Date: 2015-08-12 18:23 +0200 http://bitbucket.org/pypy/pypy/changeset/3371b0e4ea3a/ Log: fix some tests diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -4907,23 +4907,22 @@ def test_setgetarrayitem_raw(self): ops = """ - [p4, p7, i30] + [p4, i7, i30] p16 = getfield_gc_r(p4, descr=valuedescr) guard_value(p16, ConstPtr(myptr), descr=) [] - p17 = getarrayitem_gc_r(p4, 1, descr=arraydescr) - i1 = getarrayitem_raw_i(p7, 1, descr=arraydescr) + i1 = getarrayitem_raw_i(i7, 1, descr=arraydescr) i2 = int_add(i1, i30) - setarrayitem_raw(p7, 1, 7, descr=arraydescr) - setarrayitem_raw(p7, 1, i2, descr=arraydescr) - jump(p4, p7, i30) - """ - expected = """ - [p4, p7, i30] - i1 = getarrayitem_raw_i(p7, 1, descr=arraydescr) + setarrayitem_raw(i7, 1, 7, descr=arraydescr) + setarrayitem_raw(i7, 1, i2, descr=arraydescr) + jump(p4, i7, i30) + """ + expected = """ + [p4, i7, i30] + i1 = getarrayitem_raw_i(i7, 1, descr=arraydescr) i2 = int_add(i1, i30) - setarrayitem_raw(p7, 1, 7, descr=arraydescr) - setarrayitem_raw(p7, 1, i2, descr=arraydescr) - jump(p4, p7, i30) + setarrayitem_raw(i7, 1, 7, descr=arraydescr) + setarrayitem_raw(i7, 1, i2, descr=arraydescr) + jump(p4, i7, i30) """ self.optimize_loop(ops, expected, ops) @@ -5987,9 +5986,9 @@ i91 = int_add(i80, 1) setfield_gc(p75, i91, descr=inst_index) - p110 = same_as(ConstPtr(myptr)) - i112 = same_as(3) - i114 = same_as(39) + p110 = same_as_r(ConstPtr(myptr)) + i112 = same_as_i(3) + i114 = same_as_i(39) jump(p0, p1, p110, p3, i112, p5, i114, p7, p8, p75, p14) """ expected = """ @@ -6041,20 +6040,10 @@ """ self.optimize_loop(ops, expected, preamble) - def test_inputargs_added_by_forcing_jumpargs(self): - # FXIME: Can this occur? - ops = """ - [p0, p1, pinv] - i1 = getfield_gc_i(pinv, descr=valuedescr) - p2 = new_with_vtable(descr=nodesize) - setfield_gc(p2, i1, descr=nextdescr) - """ - py.test.skip("no test here") - def test_immutable_not(self): ops = """ [] - p0 = new_with_vtable(ConstClass(intobj_noimmut_vtable)) + p0 = new_with_vtable(descr=noimmut_descr) setfield_gc(p0, 42, descr=noimmut_intval) escape_n(p0) jump() @@ -6064,7 +6053,7 @@ def test_immutable_variable(self): ops = """ [i0] - p0 = new_with_vtable(ConstClass(intobj_immut_vtable)) + p0 = new_with_vtable(descr=immut_descr) setfield_gc(p0, i0, descr=immut_intval) escape_n(p0) jump(i0) @@ -6074,7 +6063,7 @@ def test_immutable_incomplete(self): ops = """ [] - p0 = new_with_vtable(ConstClass(intobj_immut_vtable)) + p0 = new_with_vtable(descr=immut_descr) escape_n(p0) jump() """ @@ -6083,7 +6072,7 @@ def test_immutable_constantfold(self): ops = """ [] - p0 = new_with_vtable(ConstClass(intobj_immut_vtable)) + p0 = new_with_vtable(descr=immut_descr) setfield_gc(p0, 1242, descr=immut_intval) escape_n(p0) jump() @@ -6141,7 +6130,7 @@ ops = """ [p0] i2 = getfield_gc_pure_i(p0, descr=immut_intval) - p1 = new_with_vtable(ConstClass(intobj_immut_vtable)) + p1 = new_with_vtable(descr=immut_descr) setfield_gc(p1, 1242, descr=immut_intval) jump(p1) """ diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -161,6 +161,8 @@ intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True) noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval') immut_intval = cpu.fielddescrof(INTOBJ_IMMUT, 'intval') + noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, True) + immut_descr = cpu.sizeof(INTOBJ_IMMUT, True) PTROBJ_IMMUT = lltype.GcStruct('PTROBJ_IMMUT', ('parent', OBJECT), ('ptrval', lltype.Ptr(OBJECT)), From noreply at buildbot.pypy.org Wed Aug 12 18:56:15 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 18:56:15 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix some tests, next stop thinking Message-ID: <20150812165615.84EC51C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78947:d3f41d3f2573 Date: 2015-08-12 18:56 +0200 http://bitbucket.org/pypy/pypy/changeset/d3f41d3f2573/ Log: fix some tests, next stop thinking diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -365,7 +365,8 @@ if realclassbox is not None: assert realclassbox.same_constant(expectedclassbox) return - self.make_constant_class(op.getarg(0), expectedclassbox) + self.make_constant_class(op.getarg(0), expectedclassbox, + update_last_guard=False) def optimize_GUARD_CLASS(self, op): expectedclassbox = op.getarg(1) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7230,7 +7230,7 @@ def test_constant_getfield1(self): ops = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) + p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr) guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) call_n(123, p25, descr=nonwritedescr) @@ -7238,23 +7238,23 @@ """ preamble = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) + p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr) guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) - p123 = same_as_r(p25) - jump(p25, p187, i184, p123) + call_n(123, p25, descr=nonwritedescr) + jump(p25, p187, i184) """ short = """ [p1, p187, i184] - p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr) - guard_value(p188, ConstPtr(myptr)) [] + p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr) + #guard_value(p188, ConstPtr(myptr)) [] p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr) jump(p25) """ expected = """ - [p25, p187, i184, p189] - call_n(123, p189, descr=nonwritedescr) - jump(p25, p187, i184, p189) + [p25, p187, i184] + call_n(123, p25, descr=nonwritedescr) + jump(p25, p187, i184) """ self.optimize_loop(ops, expected, preamble, expected_short=short) @@ -8507,14 +8507,14 @@ p2 = getfield_gc_r(p1, descr=nextdescr) mark_opaque_ptr(p2) guard_class(p2, ConstClass(node_vtable)) [] - i3 = getfield_gc_r(p2, descr=otherdescr) + i3 = getfield_gc_i(p2, descr=valuedescr) i4 = call_i(i3, descr=nonwritedescr) jump(p1) """ expected = """ - [p1, i3] + [p1, p2, i3] i4 = call_i(i3, descr=nonwritedescr) - jump(p1, i3) + jump(p1, p2, i3) """ self.optimize_loop(ops, expected) @@ -8539,13 +8539,13 @@ [p1] p2 = getfield_gc_r(p1, descr=nextdescr) mark_opaque_ptr(p2) - i3 = getfield_gc_i(p2, descr=otherdescr) + i3 = getfield_gc_i(p2, descr=valuedescr) i4 = call_i(i3, descr=nonwritedescr) jump(p1) """ expected = """ [p1, p2] - i3 = getfield_gc_i(p2, descr=otherdescr) + i3 = getfield_gc_i(p2, descr=valuedescr) i4 = call_i(i3, descr=nonwritedescr) jump(p1, p2) """ From noreply at buildbot.pypy.org Wed Aug 12 19:04:38 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 19:04:38 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: kill dead code Message-ID: <20150812170438.93CFA1C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78948:88cc46ecee6b Date: 2015-08-12 19:04 +0200 http://bitbucket.org/pypy/pypy/changeset/88cc46ecee6b/ Log: kill dead code diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -159,98 +159,6 @@ op.set_forwarded(None) return res - def random_garbage(self): - # WTF is the rest of this function - if not jumpop: - return - - cell_token = jumpop.getdescr() - assert isinstance(cell_token, JitCellToken) - stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), TargetToken(cell_token)) - - if jumpop.getopnum() == rop.JUMP: - if self.jump_to_already_compiled_trace(jumpop, patchguardop): - # Found a compiled trace to jump to - if self.short: - # Construct our short preamble - assert start_label - self.close_bridge(start_label) - return - - if start_label and self.jump_to_start_label(start_label, stop_label): - # Initial label matches, jump to it - vs = start_label.getdescr().virtual_state - if vs is not None: - args = vs.make_inputargs(stop_label.getarglist(), - self.optimizer) - else: - args = stop_label.getarglist() - jumpop = ResOperation(rop.JUMP, args, - descr=start_label.getdescr()) - #if self.short: - # # Construct our short preamble - # self.close_loop(start_label, jumpop, patchguardop) - #else: - start_label.getdescr().short_preamble = self.short - self.optimizer.send_extra_operation(jumpop) - return - - if cell_token.target_tokens: - limit = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.retrace_limit - if cell_token.retraced_count < limit: - cell_token.retraced_count += 1 - debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit)) - else: - debug_print("Retrace count reached, jumping to preamble") - assert cell_token.target_tokens[0].virtual_state is None - jumpop = jumpop.clone() - jumpop.setdescr(cell_token.target_tokens[0]) - self.optimizer.send_extra_operation(jumpop) - return - - # Found nothing to jump to, emit a label instead - - if self.short: - # Construct our short preamble - assert start_label - self.close_bridge(start_label) - - self.optimizer.flush() - - loop.operations = self.optimizer.get_newoperations() - if export_state: - jd_sd = self.optimizer.jitdriver_sd - try: - threshold = jd_sd.warmstate.disable_unrolling_threshold - except AttributeError: # tests only - threshold = sys.maxint - if len(loop.operations) > threshold: - if loop.operations[0].getopnum() == rop.LABEL: - # abandoning unrolling, too long - new_descr = stop_label.getdescr() - if loop.operations[0].getopnum() == rop.LABEL: - new_descr = loop.operations[0].getdescr() - stop_label = stop_label.copy_and_change(rop.JUMP, - descr=new_descr) - self.optimizer.send_extra_operation(stop_label) - loop.operations = self.optimizer.get_newoperations() - return None - final_state = self.export_state(start_label, stop_label) - else: - final_state = None - loop.operations.append(stop_label) - return final_state - - def jump_to_start_label(self, start_label, stop_label): - if not start_label or not stop_label: - return False - - stop_target = stop_label.getdescr() - start_target = start_label.getdescr() - assert isinstance(stop_target, TargetToken) - assert isinstance(start_target, TargetToken) - return stop_target.targeting_jitcell_token is start_target.targeting_jitcell_token - def export_state(self, start_label, end_label, renamed_inputargs): original_label_args = end_label.getarglist() end_args = [self.get_box_replacement(a) for a in original_label_args] @@ -278,56 +186,9 @@ short_boxes, renamed_inputargs, short_inputargs) - - inputargs = virtual_state.make_inputargs(jump_args, self.optimizer) - short_inputargs = virtual_state.make_inputargs(jump_args, - self.optimizer, keyboxes=True) - - #proven_constants = [] - #for i in range(len(original_jump_args)): - # srcbox = jump_args[i] - ## if srcbox is not original_jump_args[i]: - # if srcbox.type == 'r': - # info = self.optimizer.getptrinfo(srcbox) - # if info and info.is_virtual(): - # xxx - # if original_jump_args[i] is not srcbox and srcbox.is_constant(): - # proven_constants.append((original_jump_args[i], srcbox)) - #opnum = OpHelpers.same_as_for_type(original_jump_args[i].type) - #op = ResOperation(opnum, [srcbox]) - #self.optimizer.emit_operation(op) - - # if srcbox.type != 'r': - # continue - # info = self.optimizer.getptrinfo(srcbox) - # if info and info.is_virtual(): - # xxx - # srcbox = values[i].force_box(self.optimizer) - # if original_jump_args[i] is not srcbox: - # opnum = OpHelpers.same_as_for_type(original_jump_args[i].type) - # op = self.optimizer.replace_op_with(original_jump_args[i], - # opnum, [srcbox], - # descr=DONT_CHANGE) - # self.optimizer.emit_operation(op) - #inputarg_setup_ops = original_jump_args - #inputarg_setup_ops = self.optimizer.get_newoperations() - - target_token.short_preamble = [ResOperation(rop.LABEL, short_inputargs, None)] - - #exported_values = {} - #for box in inputargs: - # exported_values[box] = self.optimizer.getinfo(box) - #for op in short_boxes.operations(): - # if op and op.type != 'v': - # exported_values[op] = self.optimizer.getinfo(op) - - return ExportedState([], []) - def import_state(self, targetop, exported_state): # the mapping between input args (from old label) and what we need # to actually emit. Update the info - #self.ops_to_import = {} - virtual_state = exported_state.virtual_state for source, target in exported_state.inputarg_mapping: if source is not target: source.set_forwarded(target) @@ -343,254 +204,6 @@ for produced_op in exported_state.short_boxes: produced_op.produce_op(self, exported_state.exported_infos) - return - self.inputargs = targetop.getarglist() - target_token = targetop.getdescr() - assert isinstance(target_token, TargetToken) - if not exported_state: - # No state exported, construct one without virtuals - self.short = None - virtual_state = self.get_virtual_state(self.inputargs) - self.initial_virtual_state = virtual_state - return - - self.short = [] # target_token.short_preamble[:] - #self.short_seen = {} - self.initial_virtual_state = target_token.virtual_state - - #inpargs = self.initial_virtual_state.make_inputargs( - # exported_state.orig_inputargs, self.optimizer) - #for i, arg in enumerate(inpargs): - # if arg is not self.inputargs[i]: - # arg.set_forwarded(self.inputargs[i]) - #for box in self.inputargs: - # preamble_info = exported_state.exported_values[box] - # self.optimizer.setinfo_from_preamble(box, preamble_info) - #for box, const in exported_state.state: - # box.set_forwarded(const) - - # Setup the state of the new optimizer by emiting the - # short operations and discarding the result - #self.optimizer.emitting_dissabled = True - # think about it, it seems to be just for consts - #for source, target in exported_state.inputarg_setup_ops: - # source.set_forwarded(target) - for source, target in exported_state.inputarg_mapping: - source.set_forwarded(target) - - for op, preamble_op in exported_state.short_boxes.iteritems(): - if preamble_op.is_always_pure(): - self.pure(op.getopnum(), PreambleOp(op, None)) - else: - xxx - return - seen = {} - for op in self.short_boxes.operations(): - yyy - self.ensure_short_op_emitted(op, self.optimizer, seen) - if op and op.type != 'v': - preamble_value = exported_state.exported_values[op] - continue - value = self.optimizer.getvalue(op) - if not value.is_virtual() and not value.is_constant(): - imp = ValueImporter(self, preamble_value, op) - self.optimizer.importable_values[value] = imp - newvalue = self.optimizer.getvalue(op) - newresult = newvalue.get_key_box() - # note that emitting here SAME_AS should not happen, but - # in case it does, we would prefer to be suboptimal in asm - # to a fatal RPython exception. - # XXX investigate what is it - xxxx - if source_op is not op and \ - not self.short_boxes.has_producer(newresult) and \ - not newvalue.is_constant(): - xxx - opnum = OpHelpers.same_as_for_type(op.type) - op = ResOperation(opnum, [op]) - self.optimizer._newoperations.append(op) - #if self.optimizer.loop.logops: - # debug_print(' Falling back to add extra: ' + - # self.optimizer.loop.logops.repr_of_resop(op)) - - #self.optimizer.flush() - #self.optimizer.emitting_dissabled = False - - def close_bridge(self, start_label): - inputargs = self.inputargs - short_jumpargs = inputargs[:] - - # We dont need to inline the short preamble we are creating as we are conneting - # the bridge to a different trace with a different short preamble - self.memo = None - - newoperations = self.optimizer.get_newoperations() - self.boxes_created_this_iteration = {} - i = 0 - while i < len(newoperations): - self._import_op(newoperations[i], inputargs, short_jumpargs, []) - i += 1 - newoperations = self.optimizer.get_newoperations() - self.short.append(ResOperation(rop.JUMP, short_jumpargs, None, descr=start_label.getdescr())) - self.finalize_short_preamble(start_label) - - def close_loop(self, start_label, jumpop, patchguardop): - virtual_state = self.initial_virtual_state - short_inputargs = self.short[0].getarglist() - inputargs = self.inputargs - short_jumpargs = inputargs[:] - - # Construct jumpargs from the virtual state - original_jumpargs = jumpop.getarglist()[:] - jump_boxes = [self.get_box_replacement(arg) for arg in - jumpop.getarglist()] - try: - jumpargs = virtual_state.make_inputargs(jump_boxes, self.optimizer) - except BadVirtualState: - raise InvalidLoop('The state of the optimizer at the end of ' + - 'peeled loop is inconsistent with the ' + - 'VirtualState at the beginning of the peeled ' + - 'loop') - jumpop.initarglist(jumpargs) - - # Inline the short preamble at the end of the loop - jmp_to_short_args = virtual_state.make_inputargs(jump_boxes, - self.optimizer, - keyboxes=True) - assert len(short_inputargs) == len(jmp_to_short_args) - args = {} - for i in range(len(short_inputargs)): - if short_inputargs[i] in args: - if args[short_inputargs[i]] != jmp_to_short_args[i]: - raise InvalidLoop('The short preamble wants the ' + - 'same box passed to multiple of its ' + - 'inputargs, but the jump at the ' + - 'end of this bridge does not do that.') - - args[short_inputargs[i]] = jmp_to_short_args[i] - self.memo = Memo(short_inputargs, jmp_to_short_args) - self._inline_short_preamble(self.short, self.memo, - patchguardop, - self.short_boxes.assumed_classes) - - # Import boxes produced in the preamble but used in the loop - newoperations = self.optimizer.get_newoperations() - self.boxes_created_this_iteration = {} - i = j = 0 - while i < len(newoperations) or j < len(jumpargs): - if i == len(newoperations): - while j < len(jumpargs): - a = jumpargs[j] - #if self.optimizer.loop.logops: - # debug_print('J: ' + self.optimizer.loop.logops.repr_of_arg(a)) - self.import_box(a, inputargs, short_jumpargs, jumpargs) - j += 1 - else: - self._import_op(newoperations[i], inputargs, short_jumpargs, jumpargs) - i += 1 - newoperations = self.optimizer.get_newoperations() - - jumpop.initarglist(jumpargs) - self.optimizer.send_extra_operation(jumpop) - self.short.append(ResOperation(rop.JUMP, short_jumpargs, descr=jumpop.getdescr())) - - # Verify that the virtual state at the end of the loop is one - # that is compatible with the virtual state at the start of the loop - final_virtual_state = self.get_virtual_state(original_jumpargs) - #debug_start('jit-log-virtualstate') - #virtual_state.debug_print('Closed loop with ') - bad = {} - if not virtual_state.generalization_of(final_virtual_state, bad, - cpu=self.optimizer.cpu): - # We ended up with a virtual state that is not compatible - # and we are thus unable to jump to the start of the loop - #final_virtual_state.debug_print("Bad virtual state at end of loop, ", - # bad) - #debug_stop('jit-log-virtualstate') - raise InvalidLoop('The virtual state at the end of the peeled ' + - 'loop is not compatible with the virtual ' + - 'state at the start of the loop which makes ' + - 'it impossible to close the loop') - - #debug_stop('jit-log-virtualstate') - - maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards - if self.optimizer.emitted_guards > maxguards: - target_token = jumpop.getdescr() - assert isinstance(target_token, TargetToken) - target_token.targeting_jitcell_token.retraced_count = sys.maxint - - self.finalize_short_preamble(start_label) - - def finalize_short_preamble(self, start_label): - short = self.short - assert short[-1].getopnum() == rop.JUMP - target_token = start_label.getdescr() - assert isinstance(target_token, TargetToken) - # Turn guards into conditional jumps to the preamble - #for i in range(len(short)): - # op = short[i] - # if op.is_guard(): - #op = op.clone(self.memo) - #op.is_source_op = True - # op.setfailargs(None) - # op.setdescr(None) # will be set to a proper descr when the preamble is used - # short[i] = op - - # Clone ops and boxes to get private versions and - return - short_inputargs = short[0].getarglist() - boxmap = {} - newargs = [None] * len(short_inputargs) - for i in range(len(short_inputargs)): - a = short_inputargs[i] - if a in boxmap: - newargs[i] = boxmap[a] - else: - newargs[i] = a.clone_input_arg() - boxmap[a] = newargs[i] - #memo = Memo(short_inputargs, newargs) - #target_token.assumed_classes = {} - for i in range(len(short)): - op = short[i] - newop = op.clone(memo) - if newop.is_guard(): - newop.setfailargs(None) - newop.setdescr(None) - if op in self.short_boxes.assumed_classes: - target_token.assumed_classes[newop] = self.short_boxes.assumed_classes[op] - short[i] = newop - - # Forget the values to allow them to be freed - for box in short[0].getarglist(): - box.forget_value() - for op in short: - op.forget_value() - target_token.short_preamble = self.short - - def ensure_short_op_emitted(self, op, optimizer, seen): - if op is None: - return - if op.type != 'v' and op in seen: - return - for a in op.getarglist(): - if not isinstance(a, Const) and not isinstance(a, AbstractInputArg) and a not in seen: - self.ensure_short_op_emitted(self.short_boxes.producer(a), optimizer, - seen) - - #if self.optimizer.loop.logops: - # debug_print(' Emitting short op: ' + - # self.optimizer.loop.logops.repr_of_resop(op)) - - optimizer.send_extra_operation(op) - seen[op] = None - if op.is_ovf(): - guard = ResOperation(rop.GUARD_NO_OVERFLOW, []) - optimizer.send_extra_operation(guard) - if self.is_call_pure_with_exception(op): # only for MemoryError - guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) - optimizer.send_extra_operation(guard) - def is_call_pure_with_exception(self, op): if op.is_call_pure(): effectinfo = op.getdescr().get_extra_info() @@ -603,177 +216,6 @@ return effectinfo.extraeffect != effectinfo.EF_ELIDABLE_CANNOT_RAISE return False - def add_op_to_short(self, op, emit=True, guards_needed=False): - if op is None: - return None - if op is not None and op in self.short_seen: - if emit and self.memo: - return self.memo.get(op, op) - else: - return None - - for a in op.getarglist(): - if not isinstance(a, Const) and a not in self.short_seen: - self.add_op_to_short(self.short_boxes.producer(a), emit, guards_needed) - if op.is_guard(): - op.setdescr(None) # will be set to a proper descr when the preamble is used - - if guards_needed and self.short_boxes.has_producer(op): - value_guards = self.getvalue(op).make_guards(op) - else: - value_guards = [] - - self.short.append(op) - self.short_seen[op] = None - if emit and self.short_inliner: - newop = self.short_inliner.inline_op(op) - self.optimizer.send_extra_operation(newop) - else: - newop = None - - if op.is_ovf(): - # FIXME: ensure that GUARD_OVERFLOW:ed ops not end up here - guard = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - self.add_op_to_short(guard, emit, guards_needed) - if self.is_call_pure_with_exception(op): # only for MemoryError - guard = ResOperation(rop.GUARD_NO_EXCEPTION, [], None) - self.add_op_to_short(guard, emit, guards_needed) - for guard in value_guards: - self.add_op_to_short(guard, emit, guards_needed) - - if newop: - return newop.result - return None - - def import_box(self, box, inputargs, short_jumpargs, jumpargs): - if isinstance(box, Const) or box in inputargs: - return - if box in self.boxes_created_this_iteration: - return - - short_op = self.short_boxes.producer(box) - newresult = self.add_op_to_short(short_op) - - short_jumpargs.append(short_op) - inputargs.append(box) - box = newresult - if box in self.optimizer.values: - box = self.optimizer.values[box].force_box(self.optimizer) - jumpargs.append(box) - - - def _import_op(self, op, inputargs, short_jumpargs, jumpargs): - self.boxes_created_this_iteration[op] = None - args = op.getarglist() - if op.is_guard(): - args = args + op.getfailargs() - - for a in args: - self.import_box(a, inputargs, short_jumpargs, jumpargs) - - def jump_to_already_compiled_trace(self, jumpop, patchguardop): - jumpop = jumpop.copy_and_change(jumpop.getopnum()) - assert jumpop.getopnum() == rop.JUMP - cell_token = jumpop.getdescr() - - assert isinstance(cell_token, JitCellToken) - if not cell_token.target_tokens: - return False - - if not self.inline_short_preamble: - assert cell_token.target_tokens[0].virtual_state is None - jumpop.setdescr(cell_token.target_tokens[0]) - self.optimizer.send_extra_operation(jumpop) - return True - - args = jumpop.getarglist() - virtual_state = self.get_virtual_state(args) - values = [self.getvalue(arg) - for arg in jumpop.getarglist()] - debug_start('jit-log-virtualstate') - virtual_state.debug_print("Looking for ", metainterp_sd=self.optimizer.metainterp_sd) - - for target in cell_token.target_tokens: - if not target.virtual_state: - continue - extra_guards = [] - - try: - cpu = self.optimizer.cpu - state = target.virtual_state.generate_guards(virtual_state, - values, - cpu) - - extra_guards = state.extra_guards - if extra_guards: - debugmsg = 'Guarded to match ' - else: - debugmsg = 'Matched ' - except VirtualStatesCantMatch, e: - debugmsg = 'Did not match:\n%s\n' % (e.msg, ) - target.virtual_state.debug_print(debugmsg, e.state.bad, metainterp_sd=self.optimizer.metainterp_sd) - continue - - assert patchguardop is not None or (extra_guards == [] and len(target.short_preamble) == 1) - - target.virtual_state.debug_print(debugmsg, {}) - - debug_stop('jit-log-virtualstate') - - args = target.virtual_state.make_inputargs(values, self.optimizer, - keyboxes=True) - short_inputargs = target.short_preamble[0].getarglist() - memo = Memo(short_inputargs, args) - - for guard in extra_guards: - if guard.is_guard(): - assert isinstance(patchguardop, GuardResOp) - assert isinstance(guard, GuardResOp) - guard.rd_snapshot = patchguardop.rd_snapshot - guard.rd_frame_info_list = patchguardop.rd_frame_info_list - guard.setdescr(compile.ResumeAtPositionDescr()) - self.optimizer.send_extra_operation(guard) - - try: - # NB: the short_preamble ends with a jump - self._inline_short_preamble(target.short_preamble, memo, - patchguardop, - target.assumed_classes) - except InvalidLoop: - #debug_print("Inlining failed unexpectedly", - # "jumping to preamble instead") - assert cell_token.target_tokens[0].virtual_state is None - jumpop.setdescr(cell_token.target_tokens[0]) - self.optimizer.send_extra_operation(jumpop) - return True - debug_stop('jit-log-virtualstate') - return False - - def _inline_short_preamble(self, short_preamble, memo, patchguardop, - assumed_classes): - i = 1 - # XXX this is intentiontal :-(. short_preamble can change during the - # loop in some cases - while i < len(short_preamble): - shop = short_preamble[i] - newop = shop.clone(memo) - if newop.is_guard(): - if not patchguardop: - raise InvalidLoop("would like to have short preamble, but it has a guard and there's no guard_future_condition") - assert isinstance(newop, GuardResOp) - assert isinstance(patchguardop, GuardResOp) - newop.rd_snapshot = patchguardop.rd_snapshot - newop.rd_frame_info_list = patchguardop.rd_frame_info_list - newop.setdescr(compile.ResumeAtPositionDescr()) - self.optimizer.send_extra_operation(newop) - if shop in assumed_classes: - classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu) - if not classbox or not classbox.same_constant(assumed_classes[shop.result]): - raise InvalidLoop('The class of an opaque pointer before the jump ' + - 'does not mach the class ' + - 'it has at the start of the target loop') - i += 1 - class UnrollInfo(LoopInfo): """ A state after optimizing the peeled loop, contains the following: From noreply at buildbot.pypy.org Wed Aug 12 19:10:42 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 19:10:42 +0200 (CEST) Subject: [pypy-commit] pypy default: (cfbolz, arigo and fijal around): change record_known_class to record_exact_class Message-ID: <20150812171042.6F51C1C0726@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r78949:2e627a5cee8b Date: 2015-08-12 19:10 +0200 http://bitbucket.org/pypy/pypy/changeset/2e627a5cee8b/ Log: (cfbolz, arigo and fijal around): change record_known_class to record_exact_class diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -252,9 +252,11 @@ TO = op.result.concretetype.TO if lltype._castdepth(TO, FROM) > 0: vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, TO) - const_vtable = Constant(vtable, lltype.typeOf(vtable)) - return [None, # hack, do the right renaming from op.args[0] to op.result - SpaceOperation("record_known_class", [op.args[0], const_vtable], None)] + if vtable.subclassrange_max - vtable.subclassrange_min == 1: + # it's a precise class check + const_vtable = Constant(vtable, lltype.typeOf(vtable)) + return [None, # hack, do the right renaming from op.args[0] to op.result + SpaceOperation("record_exact_class", [op.args[0], const_vtable], None)] def rewrite_op_likely(self, op): return None # "no real effect" @@ -271,8 +273,8 @@ arg = llmemory.raw_malloc_usage(arg) return [Constant(arg, lltype.Signed)] - def rewrite_op_jit_record_known_class(self, op): - return SpaceOperation("record_known_class", [op.args[0], op.args[1]], None) + def rewrite_op_jit_record_exact_class(self, op): + return SpaceOperation("record_exact_class", [op.args[0], op.args[1]], None) def rewrite_op_cast_bool_to_int(self, op): pass def rewrite_op_cast_bool_to_uint(self, op): pass diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -536,7 +536,7 @@ def bhimpl_mark_opaque_ptr(a): pass @arguments("r", "i") - def bhimpl_record_known_class(a, b): + def bhimpl_record_exact_class(a, b): pass @arguments("i", returns="i") diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -331,7 +331,7 @@ def optimize_GUARD_FALSE(self, op): self.optimize_guard(op, CONST_0) - def optimize_RECORD_KNOWN_CLASS(self, op): + def optimize_RECORD_EXACT_CLASS(self, op): value = self.getvalue(op.getarg(0)) expectedclassbox = op.getarg(1) assert isinstance(expectedclassbox, Const) diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py --- a/rpython/jit/metainterp/optimizeopt/simplify.py +++ b/rpython/jit/metainterp/optimizeopt/simplify.py @@ -35,7 +35,7 @@ # but it's a bit hard to implement robustly if heap.py is also run pass - def optimize_RECORD_KNOWN_CLASS(self, op): + def optimize_RECORD_EXACT_CLASS(self, op): pass def optimize_LABEL(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7084,11 +7084,11 @@ # not obvious, because of the exception UnicodeDecodeError that # can be raised by ll_str2unicode() - def test_record_known_class(self): + def test_record_exact_class(self): ops = """ [p0] p1 = getfield_gc(p0, descr=nextdescr) - record_known_class(p1, ConstClass(node_vtable)) + record_exact_class(p1, ConstClass(node_vtable)) guard_class(p1, ConstClass(node_vtable)) [] jump(p1) """ diff --git a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py --- a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py @@ -30,7 +30,7 @@ rop.VIRTUAL_REF, rop.QUASIIMMUT_FIELD, rop.MARK_OPAQUE_PTR, - rop.RECORD_KNOWN_CLASS) + rop.RECORD_EXACT_CLASS) def raises(self, e, fn, *args): try: diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -275,16 +275,13 @@ return self.execute(rop.MARK_OPAQUE_PTR, box) @arguments("box", "box") - def opimpl_record_known_class(self, box, clsbox): + def opimpl_record_exact_class(self, box, clsbox): from rpython.rtyper.lltypesystem import llmemory if self.metainterp.heapcache.is_class_known(box): return adr = clsbox.getaddr() - bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) - if bounding_class.subclassrange_max - bounding_class.subclassrange_min == 1: - # precise class knowledge, this can be used - self.execute(rop.RECORD_KNOWN_CLASS, box, clsbox) - self.metainterp.heapcache.class_now_known(box) + self.execute(rop.RECORD_EXACT_CLASS, box, clsbox) + self.metainterp.heapcache.class_now_known(box) @arguments("box") def _opimpl_any_return(self, box): diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -529,7 +529,7 @@ 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length 'COPYUNICODECONTENT/5', 'QUASIIMMUT_FIELD/1d', # [objptr], descr=SlowMutateDescr - 'RECORD_KNOWN_CLASS/2', # [objptr, clsptr] + 'RECORD_EXACT_CLASS/2', # [objptr, clsptr] 'KEEPALIVE/1', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -10,7 +10,7 @@ from rpython.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside, loop_invariant, elidable, promote, jit_debug, assert_green, AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff, - isconstant, isvirtual, set_param, record_known_class) + isconstant, isvirtual, set_param, record_exact_class) from rpython.rlib.longlong2float import float2longlong, longlong2float from rpython.rlib.rarithmetic import ovfcheck, is_valid_int, int_force_ge_zero from rpython.rtyper.lltypesystem import lltype, rffi @@ -3756,19 +3756,19 @@ res1 = f(6) res2 = self.interp_operations(f, [6]) assert res1 == res2 - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) res1 = f(-6) res2 = self.interp_operations(f, [-6]) assert res1 == res2 - # cannot use record_known_class here, because B has a subclass + # cannot use record_exact_class here, because B has a subclass self.check_operations_history(guard_class=1) res1 = f(0) res2 = self.interp_operations(f, [0]) assert res1 == res2 # here it works again - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) def test_give_class_knowledge_to_tracer_explicitly(self): from rpython.rtyper.lltypesystem.lloperation import llop @@ -3808,31 +3808,30 @@ def f(x): a = make(x) if x > 0: - record_known_class(a, A) + record_exact_class(a, A) z = a.f() elif x < 0: - record_known_class(a, B) + record_exact_class(a, B) z = a.f() else: - record_known_class(a, C) + record_exact_class(a, C) z = a.f() return z + a.g() res1 = f(6) res2 = self.interp_operations(f, [6]) assert res1 == res2 - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) res1 = f(-6) res2 = self.interp_operations(f, [-6]) assert res1 == res2 - # cannot use record_known_class here, because B has a subclass - self.check_operations_history(guard_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) res1 = f(0) res2 = self.interp_operations(f, [0]) assert res1 == res2 # here it works again - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) def test_generator(self): def g(n): diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1080,19 +1080,18 @@ instance, overwrite for custom behavior """ -def record_known_class(value, cls): +def record_exact_class(value, cls): """ - Assure the JIT that value is an instance of cls. This is not a precise - class check, unlike a guard_class. + Assure the JIT that value is an instance of cls. This is a precise + class check, like a guard_class. """ - assert isinstance(value, cls) + assert type(value) is cls class Entry(ExtRegistryEntry): - _about_ = record_known_class + _about_ = record_exact_class def compute_result_annotation(self, s_inst, s_cls): from rpython.annotator import model as annmodel - assert s_cls.is_constant() assert not s_inst.can_be_none() assert isinstance(s_inst, annmodel.SomeInstance) @@ -1105,7 +1104,7 @@ hop.exception_cannot_occur() v_inst = hop.inputarg(hop.args_r[0], arg=0) v_cls = hop.inputarg(classrepr, arg=1) - return hop.genop('jit_record_known_class', [v_inst, v_cls], + return hop.genop('jit_record_exact_class', [v_inst, v_cls], resulttype=lltype.Void) def _jit_conditional_call(condition, function, *args): diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -545,7 +545,7 @@ def op_jit_marker(self, *args): pass - def op_jit_record_known_class(self, *args): + def op_jit_record_exact_class(self, *args): pass def op_jit_conditional_call(self, *args): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -449,7 +449,7 @@ 'jit_force_virtual': LLOp(canrun=True), 'jit_is_virtual': LLOp(canrun=True), 'jit_force_quasi_immutable': LLOp(canrun=True), - 'jit_record_known_class' : LLOp(canrun=True), + 'jit_record_exact_class' : LLOp(canrun=True), 'jit_ffi_save_result': LLOp(canrun=True), 'jit_conditional_call': LLOp(), 'get_exception_addr': LLOp(), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -618,7 +618,7 @@ def op_jit_force_quasi_immutable(*args): pass -def op_jit_record_known_class(x, y): +def op_jit_record_exact_class(x, y): pass def op_jit_ffi_save_result(*args): diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h --- a/rpython/translator/c/src/support.h +++ b/rpython/translator/c/src/support.h @@ -6,7 +6,7 @@ #define _SRC_SUPPORT_H #define RUNNING_ON_LLINTERP 0 -#define OP_JIT_RECORD_KNOWN_CLASS(i, c, r) /* nothing */ +#define OP_JIT_RECORD_EXACT_CLASS(i, c, r) /* nothing */ #define FAIL_OVF(msg) _RPyRaiseSimpleException(RPyExc_OverflowError) #define FAIL_VAL(msg) _RPyRaiseSimpleException(RPyExc_ValueError) From noreply at buildbot.pypy.org Wed Aug 12 19:15:25 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 19:15:25 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: merge default Message-ID: <20150812171525.7C2501C0726@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78950:ae5f520acf4a Date: 2015-08-12 19:14 +0200 http://bitbucket.org/pypy/pypy/changeset/ae5f520acf4a/ Log: merge default diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -252,9 +252,11 @@ TO = op.result.concretetype.TO if lltype._castdepth(TO, FROM) > 0: vtable = heaptracker.get_vtable_for_gcstruct(self.cpu, TO) - const_vtable = Constant(vtable, lltype.typeOf(vtable)) - return [None, # hack, do the right renaming from op.args[0] to op.result - SpaceOperation("record_known_class", [op.args[0], const_vtable], None)] + if vtable.subclassrange_max - vtable.subclassrange_min == 1: + # it's a precise class check + const_vtable = Constant(vtable, lltype.typeOf(vtable)) + return [None, # hack, do the right renaming from op.args[0] to op.result + SpaceOperation("record_exact_class", [op.args[0], const_vtable], None)] def rewrite_op_likely(self, op): return None # "no real effect" @@ -271,8 +273,8 @@ arg = llmemory.raw_malloc_usage(arg) return [Constant(arg, lltype.Signed)] - def rewrite_op_jit_record_known_class(self, op): - return SpaceOperation("record_known_class", [op.args[0], op.args[1]], None) + def rewrite_op_jit_record_exact_class(self, op): + return SpaceOperation("record_exact_class", [op.args[0], op.args[1]], None) def rewrite_op_cast_bool_to_int(self, op): pass def rewrite_op_cast_bool_to_uint(self, op): pass diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -536,7 +536,7 @@ def bhimpl_mark_opaque_ptr(a): pass @arguments("r", "i") - def bhimpl_record_known_class(a, b): + def bhimpl_record_exact_class(a, b): pass @arguments("i", returns="i") diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -331,7 +331,7 @@ def optimize_GUARD_FALSE(self, op): self.optimize_guard(op, CONST_0) - def optimize_RECORD_KNOWN_CLASS(self, op): + def optimize_RECORD_EXACT_CLASS(self, op): value = self.getvalue(op.getarg(0)) expectedclassbox = op.getarg(1) assert isinstance(expectedclassbox, Const) diff --git a/rpython/jit/metainterp/optimizeopt/simplify.py b/rpython/jit/metainterp/optimizeopt/simplify.py --- a/rpython/jit/metainterp/optimizeopt/simplify.py +++ b/rpython/jit/metainterp/optimizeopt/simplify.py @@ -35,7 +35,7 @@ # but it's a bit hard to implement robustly if heap.py is also run pass - def optimize_RECORD_KNOWN_CLASS(self, op): + def optimize_RECORD_EXACT_CLASS(self, op): pass def optimize_LABEL(self, op): diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -7084,11 +7084,11 @@ # not obvious, because of the exception UnicodeDecodeError that # can be raised by ll_str2unicode() - def test_record_known_class(self): + def test_record_exact_class(self): ops = """ [p0] p1 = getfield_gc(p0, descr=nextdescr) - record_known_class(p1, ConstClass(node_vtable)) + record_exact_class(p1, ConstClass(node_vtable)) guard_class(p1, ConstClass(node_vtable)) [] jump(p1) """ @@ -7352,7 +7352,9 @@ i3 = int_add(i1, i2) setfield_gc(p0, ii, descr=valuedescr) setfield_gc(p1, ii, descr=otherdescr) - jump(p0, p1, ii2, ii, ii, ii) + i7 = same_as(ii) + i8 = same_as(ii) + jump(p0, p1, ii2, ii, i8, i7) """ expected = """ [p0, p1, ii, ii2, i1, i2] @@ -7361,7 +7363,7 @@ setfield_gc(p1, ii, descr=otherdescr) jump(p0, p1, ii2, ii, ii, ii) """ - self.optimize_loop(ops, expected) + self.optimize_loop(ops, expected, preamble) def test_dont_specialize_on_boxes_equal(self): ops = """ diff --git a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py --- a/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_zdisable_opts.py @@ -30,7 +30,7 @@ rop.VIRTUAL_REF, rop.QUASIIMMUT_FIELD, rop.MARK_OPAQUE_PTR, - rop.RECORD_KNOWN_CLASS) + rop.RECORD_EXACT_CLASS) def raises(self, e, fn, *args): try: diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -275,16 +275,13 @@ return self.execute(rop.MARK_OPAQUE_PTR, box) @arguments("box", "box") - def opimpl_record_known_class(self, box, clsbox): + def opimpl_record_exact_class(self, box, clsbox): from rpython.rtyper.lltypesystem import llmemory if self.metainterp.heapcache.is_class_known(box): return adr = clsbox.getaddr() - bounding_class = llmemory.cast_adr_to_ptr(adr, rclass.CLASSTYPE) - if bounding_class.subclassrange_max - bounding_class.subclassrange_min == 1: - # precise class knowledge, this can be used - self.execute(rop.RECORD_KNOWN_CLASS, box, clsbox) - self.metainterp.heapcache.class_now_known(box) + self.execute(rop.RECORD_EXACT_CLASS, box, clsbox) + self.metainterp.heapcache.class_now_known(box) @arguments("box") def _opimpl_any_return(self, box): diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -529,7 +529,7 @@ 'COPYSTRCONTENT/5', # src, dst, srcstart, dststart, length 'COPYUNICODECONTENT/5', 'QUASIIMMUT_FIELD/1d', # [objptr], descr=SlowMutateDescr - 'RECORD_KNOWN_CLASS/2', # [objptr, clsptr] + 'RECORD_EXACT_CLASS/2', # [objptr, clsptr] 'KEEPALIVE/1', '_CANRAISE_FIRST', # ----- start of can_raise operations ----- diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -10,7 +10,7 @@ from rpython.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside, loop_invariant, elidable, promote, jit_debug, assert_green, AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff, - isconstant, isvirtual, set_param, record_known_class) + isconstant, isvirtual, set_param, record_exact_class) from rpython.rlib.longlong2float import float2longlong, longlong2float from rpython.rlib.rarithmetic import ovfcheck, is_valid_int, int_force_ge_zero from rpython.rtyper.lltypesystem import lltype, rffi @@ -3756,19 +3756,19 @@ res1 = f(6) res2 = self.interp_operations(f, [6]) assert res1 == res2 - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) res1 = f(-6) res2 = self.interp_operations(f, [-6]) assert res1 == res2 - # cannot use record_known_class here, because B has a subclass + # cannot use record_exact_class here, because B has a subclass self.check_operations_history(guard_class=1) res1 = f(0) res2 = self.interp_operations(f, [0]) assert res1 == res2 # here it works again - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) def test_give_class_knowledge_to_tracer_explicitly(self): from rpython.rtyper.lltypesystem.lloperation import llop @@ -3808,31 +3808,30 @@ def f(x): a = make(x) if x > 0: - record_known_class(a, A) + record_exact_class(a, A) z = a.f() elif x < 0: - record_known_class(a, B) + record_exact_class(a, B) z = a.f() else: - record_known_class(a, C) + record_exact_class(a, C) z = a.f() return z + a.g() res1 = f(6) res2 = self.interp_operations(f, [6]) assert res1 == res2 - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) res1 = f(-6) res2 = self.interp_operations(f, [-6]) assert res1 == res2 - # cannot use record_known_class here, because B has a subclass - self.check_operations_history(guard_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) res1 = f(0) res2 = self.interp_operations(f, [0]) assert res1 == res2 # here it works again - self.check_operations_history(guard_class=0, record_known_class=1) + self.check_operations_history(guard_class=0, record_exact_class=1) def test_generator(self): def g(n): diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1080,19 +1080,18 @@ instance, overwrite for custom behavior """ -def record_known_class(value, cls): +def record_exact_class(value, cls): """ - Assure the JIT that value is an instance of cls. This is not a precise - class check, unlike a guard_class. + Assure the JIT that value is an instance of cls. This is a precise + class check, like a guard_class. """ - assert isinstance(value, cls) + assert type(value) is cls class Entry(ExtRegistryEntry): - _about_ = record_known_class + _about_ = record_exact_class def compute_result_annotation(self, s_inst, s_cls): from rpython.annotator import model as annmodel - assert s_cls.is_constant() assert not s_inst.can_be_none() assert isinstance(s_inst, annmodel.SomeInstance) @@ -1105,7 +1104,7 @@ hop.exception_cannot_occur() v_inst = hop.inputarg(hop.args_r[0], arg=0) v_cls = hop.inputarg(classrepr, arg=1) - return hop.genop('jit_record_known_class', [v_inst, v_cls], + return hop.genop('jit_record_exact_class', [v_inst, v_cls], resulttype=lltype.Void) def _jit_conditional_call(condition, function, *args): diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -839,6 +839,9 @@ if key in target: raise Exception("import_from_mixin: would overwrite the value " "already defined locally for %r" % (key,)) + if key == '_mixin_': + raise Exception("import_from_mixin(M): class M should not " + "have '_mixin_ = True'") target[key] = value if immutable_fields: target['_immutable_fields_'] = target.get('_immutable_fields_', []) + immutable_fields diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -545,7 +545,7 @@ def op_jit_marker(self, *args): pass - def op_jit_record_known_class(self, *args): + def op_jit_record_exact_class(self, *args): pass def op_jit_conditional_call(self, *args): diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -449,7 +449,7 @@ 'jit_force_virtual': LLOp(canrun=True), 'jit_is_virtual': LLOp(canrun=True), 'jit_force_quasi_immutable': LLOp(canrun=True), - 'jit_record_known_class' : LLOp(canrun=True), + 'jit_record_exact_class' : LLOp(canrun=True), 'jit_ffi_save_result': LLOp(canrun=True), 'jit_conditional_call': LLOp(), 'get_exception_addr': LLOp(), diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -618,7 +618,7 @@ def op_jit_force_quasi_immutable(*args): pass -def op_jit_record_known_class(x, y): +def op_jit_record_exact_class(x, y): pass def op_jit_ffi_save_result(*args): diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h --- a/rpython/translator/c/src/support.h +++ b/rpython/translator/c/src/support.h @@ -6,7 +6,7 @@ #define _SRC_SUPPORT_H #define RUNNING_ON_LLINTERP 0 -#define OP_JIT_RECORD_KNOWN_CLASS(i, c, r) /* nothing */ +#define OP_JIT_RECORD_EXACT_CLASS(i, c, r) /* nothing */ #define FAIL_OVF(msg) _RPyRaiseSimpleException(RPyExc_OverflowError) #define FAIL_VAL(msg) _RPyRaiseSimpleException(RPyExc_ValueError) From noreply at buildbot.pypy.org Wed Aug 12 19:15:27 2015 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 12 Aug 2015 19:15:27 +0200 (CEST) Subject: [pypy-commit] pypy value-profiling: reanme to record_exact_class Message-ID: <20150812171527.9C7F61C0726@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: value-profiling Changeset: r78951:0cbfedc2259a Date: 2015-08-12 19:15 +0200 http://bitbucket.org/pypy/pypy/changeset/0cbfedc2259a/ Log: reanme to record_exact_class diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -148,7 +148,6 @@ def _getlocal(self, varindex): from pypy.objspace.std.intobject import W_IntObject # some careful logic there - vprof = None if we_are_jitted(): vprof = self.getcode().vprofs[varindex] if vprof.can_fold_read_int(): @@ -158,8 +157,10 @@ if w_res is not None: return w_res w_res = self.locals_cells_stack_w[varindex] - if we_are_jitted() and vprof.class_is_known(): - jit.record_known_class(w_res, vprof.read_constant_cls()) + if we_are_jitted(): + vprof = self.getcode().vprofs[varindex] + if vprof.class_is_known(): + jit.record_exact_class(w_res, vprof.read_constant_cls()) return w_res def _setlocal(self, varindex, value): diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -52,7 +52,7 @@ else: w_res = obj._mapdict_read_storage(attr.storageindex) if jit.we_are_jitted() and attr.class_is_known(): - jit.record_known_class(w_res, attr.read_constant_cls()) + jit.record_exact_class(w_res, attr.read_constant_cls()) return w_res @jit.elidable diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -391,7 +391,7 @@ def f(obj, cls): seen.append((obj, cls)) monkeypatch.setattr(jit, "we_are_jitted", lambda : True) - monkeypatch.setattr(jit, "record_known_class", f) + monkeypatch.setattr(jit, "record_exact_class", f) assert obj.getdictvalue(space, "a") == a assert obj.getdictvalue(space, "a") == a From noreply at buildbot.pypy.org Wed Aug 12 19:15:45 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 19:15:45 +0200 (CEST) Subject: [pypy-commit] pypy optresult-unroll: fix some tests in test_optimizebasic Message-ID: <20150812171545.538A31C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: optresult-unroll Changeset: r78952:c51373696582 Date: 2015-08-12 19:15 +0200 http://bitbucket.org/pypy/pypy/changeset/c51373696582/ Log: fix some tests in test_optimizebasic diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -2881,8 +2881,8 @@ [p1, i1, p2, i2] setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=readadescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) - setfield_gc(p2, i3, descr=adescr) jump(p1, i1, p2, i2) """ self.optimize_loop(ops, expected) @@ -2893,16 +2893,16 @@ setfield_gc(p1, i1, descr=valuedescr) setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=writeadescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) - setfield_gc(p2, i3, descr=adescr) jump(p1, i1, p2, i2) """ expected = """ [p1, i1, p2, i2] setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=writeadescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) - setfield_gc(p2, i3, descr=adescr) jump(p1, i1, p2, i2) """ self.optimize_loop(ops, expected) @@ -2913,11 +2913,20 @@ setfield_gc(p1, i1, descr=valuedescr) setfield_gc(p2, i2, descr=adescr) i3 = call_i(i1, descr=plaincalldescr) + setfield_gc(p2, i3, descr=adescr) setfield_gc(p1, i3, descr=valuedescr) + jump(p1, i1, p2, i2) + """ + expected = """ + [p1, i1, p2, i2] + setfield_gc(p2, i2, descr=adescr) + setfield_gc(p1, i1, descr=valuedescr) + i3 = call_i(i1, descr=plaincalldescr) setfield_gc(p2, i3, descr=adescr) + setfield_gc(p1, i3, descr=valuedescr) jump(p1, i1, p2, i2) """ - self.optimize_loop(ops, ops) + self.optimize_loop(ops, expected) def test_call_assembler_invalidates_caches(self): ops = ''' diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -432,7 +432,7 @@ assert equaloplists(optimized.operations, expected.operations, False, remap, text_right) - def _do_optimize_loop(self, compile_data, call_pure_results=None): + def _do_optimize_loop(self, compile_data): from rpython.jit.metainterp.optimizeopt import optimize_trace metainterp_sd = FakeMetaInterpStaticData(self.cpu) if hasattr(self, 'vrefinfo'): @@ -441,12 +441,6 @@ metainterp_sd.callinfocollection = self.callinfocollection # compile_data.enable_opts = self.enable_opts - new_call_pure_results = args_dict() - if call_pure_results is not None: - for k, v in call_pure_results.items(): - new_call_pure_results[list(k)] = v - - compile_data.call_pure_results = new_call_pure_results state = optimize_trace(metainterp_sd, None, compile_data) return state @@ -467,14 +461,16 @@ ops = loop.operations[:-1] start_label = ResOperation(rop.LABEL, loop.inputargs) end_label = jump_op.copy_and_change(opnum=rop.LABEL) - preamble_data = compile.LoopCompileData(start_label, end_label, ops) - start_state, preamble_ops = self._do_optimize_loop(preamble_data, - call_pure_results) + call_pure_results = self._convert_call_pure_results(call_pure_results) + preamble_data = compile.LoopCompileData(start_label, end_label, ops, + call_pure_results) + start_state, preamble_ops = self._do_optimize_loop(preamble_data) preamble_data.forget_optimization_info() end_label = ResOperation(rop.LABEL, start_state.end_args) loop_data = compile.UnrolledLoopData(end_label, jump_op, - ops, start_state) - loop_info, ops = self._do_optimize_loop(loop_data, call_pure_results) + ops, start_state, + call_pure_results) + loop_info, ops = self._do_optimize_loop(loop_data) preamble = TreeLoop('preamble') preamble.inputargs = start_state.renamed_inputargs start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs) From noreply at buildbot.pypy.org Wed Aug 12 19:34:15 2015 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 12 Aug 2015 19:34:15 +0200 (CEST) Subject: [pypy-commit] pypy default: (fijal, arigo) try to make vmprof C interface init a bit more lazy Message-ID: <20150812173415.D3AE81C0726@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r78953:6bfa818c8cce Date: 2015-08-12 19:34 +0200 http://bitbucket.org/pypy/pypy/changeset/6bfa818c8cce/ Log: (fijal, arigo) try to make vmprof C interface init a bit more lazy diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -6,54 +6,69 @@ from rpython.rtyper.tool import rffi_platform as platform from rpython.jit.backend import detect_cpu -if not detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64): - py.test.skip("rvmprof only supports x86-64 CPUs for now") +class VMProfPlatformUnsupported(Exception): + pass -ROOT = py.path.local(__file__).join('..') -SRC = ROOT.join('src') +def setup(): + if not detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64): + raise VMProfPlatformUnsupported("rvmprof only supports" + " x86-64 CPUs for now") -if sys.platform.startswith('linux'): - libs = ['dl'] -else: - libs = [] + ROOT = py.path.local(__file__).join('..') + SRC = ROOT.join('src') -eci_kwds = dict( - include_dirs = [SRC], - includes = ['rvmprof.h'], - libraries = libs, - separate_module_files = [SRC.join('rvmprof.c')], - post_include_bits=['#define RPYTHON_VMPROF\n'], - ) -eci = ExternalCompilationInfo(**eci_kwds) -platform.verify_eci(ExternalCompilationInfo( - compile_extra=['-DRPYTHON_LL2CTYPES'], - **eci_kwds)) + if sys.platform.startswith('linux'): + libs = ['dl'] + else: + libs = [] + eci_kwds = dict( + include_dirs = [SRC], + includes = ['rvmprof.h'], + libraries = libs, + separate_module_files = [SRC.join('rvmprof.c')], + post_include_bits=['#define RPYTHON_VMPROF\n'], + ) + eci = ExternalCompilationInfo(**eci_kwds) -vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP, - compilation_info=eci) -vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT, - compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) -vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT, - compilation_info=eci, - save_err=rffi.RFFI_SAVE_ERRNO) -vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf", - [rffi.CCHARP, rffi.LONG], - lltype.Void, compilation_info=eci) + platform.verify_eci(ExternalCompilationInfo( + compile_extra=['-DRPYTHON_LL2CTYPES'], + **eci_kwds)) -## vmprof_register_virtual_function = rffi.llexternal( -## "vmprof_register_virtual_function", -## [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, -## compilation_info=eci, _nowrapper=True) -vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals", - [rffi.INT], lltype.Void, - compilation_info=eci) + vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP, + compilation_info=eci) + vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT, + compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) + vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT, + compilation_info=eci, + save_err=rffi.RFFI_SAVE_ERRNO) + vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf", + [rffi.CCHARP, rffi.LONG], + lltype.Void, compilation_info=eci) + ## vmprof_register_virtual_function = rffi.llexternal( + ## "vmprof_register_virtual_function", + ## [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, + ## compilation_info=eci, _nowrapper=True) + + vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals", + [rffi.INT], lltype.Void, + compilation_info=eci) + return CInterface(locals()) + + +class CInterface(object): + def __init__(self, namespace): + for k, v in namespace.iteritems(): + setattr(self, k, v) + + def _freeze_(self): + return True def token2lltype(tok): if tok == 'i': diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -19,7 +19,6 @@ def __str__(self): return self.msg - class VMProf(object): def __init__(self): @@ -31,6 +30,7 @@ self._code_unique_id = 0 # XXX this is wrong, it won't work on 32bit else: self._code_unique_id = 0x7000000000000000 + self.cintf = cintf.setup() def _cleanup_(self): self.is_enabled = False @@ -106,14 +106,14 @@ raise VMProfError("bad value for 'interval'") interval_usec = int(interval * 1000000.0) - p_error = cintf.vmprof_init(fileno) + p_error = self.cintf.vmprof_init(fileno) if p_error: raise VMProfError(rffi.charp2str(p_error)) self.fileno = fileno self._write_header(interval_usec) self._gather_all_code_objs() - res = cintf.vmprof_enable(interval_usec) + res = self.cintf.vmprof_enable(interval_usec) if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) self.is_enabled = True @@ -128,7 +128,7 @@ if self._current_codes is not None: self._flush_codes() self.fileno = -1 - res = cintf.vmprof_disable() + res = self.cintf.vmprof_disable() if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) @@ -149,7 +149,7 @@ def _flush_codes(self): buf = self._current_codes.build() self._current_codes = None - cintf.vmprof_write_buf(buf, len(buf)) + self.cintf.vmprof_write_buf(buf, len(buf)) # NOTE: keep in mind that vmprof_write_buf() can only write # a maximum of 8184 bytes. This should be guaranteed here because: assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184 @@ -165,7 +165,7 @@ b.append(chr(len('pypy'))) b.append('pypy') buf = b.build() - cintf.vmprof_write_buf(buf, len(buf)) + self.cintf.vmprof_write_buf(buf, len(buf)) def _write_long_to_string_builder(l, b): From noreply at buildbot.pypy.org Wed Aug 12 22:53:42 2015 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 12 Aug 2015 22:53:42 +0200 (CEST) Subject: [pypy-commit] pypy default: test, fix unpickling 'V0' (builtin, empty) subarray in record dtype Message-ID: <20150812205342.097431C0683@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r78954:7860229d39ac Date: 2015-08-12 23:54 +0300 http://bitbucket.org/pypy/pypy/changeset/7860229d39ac/ Log: test, fix unpickling 'V0' (builtin, empty) subarray in record dtype diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -588,7 +588,8 @@ return space.newtuple([w_class, builder_args, data]) def descr_setstate(self, space, w_data): - if self.fields is None: # if builtin dtype + if self.fields is None and not isinstance(self.itemtype, types.VoidType): + # if builtin dtype (but not w_voiddtype) return space.w_None version = space.int_w(space.getitem(w_data, space.wrap(0))) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -3808,7 +3808,7 @@ assert (a == [1, 2]).all() def test_pickle(self): - from numpy import dtype, array + from numpy import dtype, array, int32 from cPickle import loads, dumps d = dtype([('x', str), ('y', 'int32')]) @@ -3825,6 +3825,11 @@ assert a[0]['y'] == 2 assert a[1]['y'] == 1 + + a = array([(1, [])], dtype=[('a', int32), ('b', int32, 0)]) + assert a['b'].shape == (1, 0) + b = loads(dumps(a)) + assert b['b'].shape == (1, 0) def test_subarrays(self): from numpy import dtype, array, zeros From noreply at buildbot.pypy.org Thu Aug 13 06:22:45 2015 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 13 Aug 2015 06:22:45 +0200 (CEST) Subject: [pypy-commit] pypy default: fix test Message-ID: <20150813042246.0083A1C11D6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r78955:7abd45f9e434 Date: 2015-08-13 07:05 +0300 http://bitbucket.org/pypy/pypy/changeset/7abd45f9e434/ Log: fix test diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -350,8 +350,8 @@ assert np.dtype(xyz).name == 'xyz' # another obscure API, used in numpy record.py # it seems numpy throws away the subclass type and parses the spec - a = np.dtype((xyz, [('x', int), ('y', float)])) - assert repr(a) == "dtype([('x', ' Author: mattip Branch: Changeset: r78956:85c33fc5132e Date: 2015-08-13 07:11 +0300 http://bitbucket.org/pypy/pypy/changeset/85c33fc5132e/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -63,3 +63,6 @@ .. branch: fix-tinylang-goals Update tinylang goals to match current rpython + +.. branch: vmprof-review + From noreply at buildbot.pypy.org Thu Aug 13 10:57:18 2015 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 13 Aug 2015 10:57:18 +0200 (CEST) Subject: [pypy-commit] buildbot default: Add a around the 
     parts
 of the html summary.
Message-ID: <20150813085718.D24FF1C1451@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r953:c413d1467f6a
Date: 2015-08-13 09:58 +0100
http://bitbucket.org/pypy/buildbot/changeset/c413d1467f6a/

Log:	Add a  around the 
     parts of the html summary.

	Avoids bad formatting if the window is not large enough.

diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py
--- a/bot2/pypybuildbot/summary.py
+++ b/bot2/pypybuildbot/summary.py
@@ -530,7 +530,7 @@
             lines.append(html.span(line,
                                    class_="a%dc%d" % (a_num, combination)))
 
-        section = html.pre(lines)
+        section = html.nobr(html.pre(lines))
         self.sections.append(section)
 
     def add_no_revision_builds(self, status, no_revision_builds):

From noreply at buildbot.pypy.org  Thu Aug 13 11:09:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 11:09:24 +0200 (CEST)
Subject: [pypy-commit] buildbot default: 2nd attempt
Message-ID: <20150813090924.3699B1C1106@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r954:a1b146e218f9
Date: 2015-08-13 10:10 +0100
http://bitbucket.org/pypy/buildbot/changeset/a1b146e218f9/

Log:	2nd attempt

diff --git a/master/public_html/default.css b/master/public_html/default.css
--- a/master/public_html/default.css
+++ b/master/public_html/default.css
@@ -566,7 +566,7 @@
 }
 
 pre {
-	white-space: pre-wrap;
+	white-space: pre;
 }
 
 /* change comments (use regular colors here) */

From noreply at buildbot.pypy.org  Thu Aug 13 11:41:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Aug 2015 11:41:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: (fijal, arigo,
 plan_rich) fix the handling of extra same_as and really test it in
 test_optimizeopt
Message-ID: <20150813094115.271E31C0221@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78957:f5479d770a54
Date: 2015-08-13 11:41 +0200
http://bitbucket.org/pypy/pypy/changeset/f5479d770a54/

Log:	(fijal, arigo, plan_rich) fix the handling of extra same_as and
	really test it in test_optimizeopt

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -250,7 +250,8 @@
                              descr=mid_descr_token)
     # XXX assign short preamble and virtual state
     loop_ops[-1].setdescr(mid_descr_token)
-    loop.operations = [start_label] + preamble_ops + [mid_label] + loop_ops
+    loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
+                       [mid_label] + loop_ops)
     loop.check_consistency()
     jitcell_token.all_target_tokens = [start_descr, mid_descr_token]
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -16,9 +16,10 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     
-    def __init__(self, op, preamble_op):
+    def __init__(self, op, preamble_op, invented_name):
         self.op = op
         self.preamble_op = preamble_op
+        self.invented_name = invented_name
 
     def numargs(self):
         return self.op.numargs()
@@ -45,7 +46,7 @@
         self.res = res
         self.getfield_op = getfield_op
 
-    def produce_op(self, opt, preamble_op, exported_infos):
+    def produce_op(self, opt, preamble_op, exported_infos, invented_name):
         optheap = opt.optimizer.optheap
         if optheap is None:
             return
@@ -56,7 +57,7 @@
                                                 exported_infos[g.getarg(0)],
                                                 exported_infos)
         opinfo = opt.optimizer.ensure_ptr_info_arg0(g)
-        pop = PreambleOp(self.res, preamble_op)
+        pop = PreambleOp(self.res, preamble_op, invented_name)
         assert not opinfo.is_virtual()
         descr = self.getfield_op.getdescr()
         if g.is_getfield():
@@ -89,15 +90,17 @@
     def __init__(self, res):
         self.res = res
 
-    def produce_op(self, opt, preamble_op, exported_infos):
+    def produce_op(self, opt, preamble_op, exported_infos, invented_name):
         optpure = opt.optimizer.optpure
         if optpure is None:
             return
         op = self.res
         if preamble_op.is_call():
-            optpure.extra_call_pure.append(PreambleOp(op, preamble_op))
+            optpure.extra_call_pure.append(PreambleOp(op, preamble_op,
+                                                      invented_name))
         else:
-            opt.pure(op.getopnum(), PreambleOp(op, preamble_op))
+            opt.pure(op.getopnum(), PreambleOp(op, preamble_op,
+                                               invented_name))
 
     def add_op_to_short(self, sb):
         op = self.res
@@ -120,13 +123,14 @@
     def __init__(self, res):
         self.res = res
 
-    def produce_op(self, opt, preamble_op, exported_infos):
+    def produce_op(self, opt, preamble_op, exported_infos, invented_name):
         optrewrite = opt.optimizer.optrewrite
         if optrewrite is None:
             return
         op = self.res
         key = make_hashable_int(op.getarg(0).getint())
-        optrewrite.loop_invariant_results[key] = PreambleOp(op, preamble_op)
+        optrewrite.loop_invariant_results[key] = PreambleOp(op, preamble_op,
+                                                            invented_name)
 
     def add_op_to_short(self, sb):
         op = self.res
@@ -165,12 +169,15 @@
     pass
 
 class ProducedShortOp(AbstractProducedShortOp):
+    invented_name = False
+    
     def __init__(self, short_op, preamble_op):
         self.short_op = short_op
         self.preamble_op = preamble_op
 
     def produce_op(self, opt, exported_infos):
-        self.short_op.produce_op(opt, self.preamble_op, exported_infos)
+        self.short_op.produce_op(opt, self.preamble_op, exported_infos,
+                                 invented_name=self.invented_name)
 
     def __repr__(self):
         return "%r -> %r" % (self.short_op, self.preamble_op)
@@ -189,10 +196,9 @@
         return "INP(%r)" % (self.preamble_op,)
 
 class ShortBoxes(object):
-    def __init__(self):
-        #self.extra_same_as = []
-        pass
-
+    """ This is a container used for creating all the exported short
+    boxes from the preamble
+    """
     def create_short_boxes(self, optimizer, inputargs, label_args):
         # all the potential operations that can be produced, subclasses
         # of AbstractShortOp
@@ -272,6 +278,7 @@
                         new_name = ResOperation(opnum, [shortop.res])
                         assert lst[i].short_op is not pop.short_op
                         lst[i].short_op.res = new_name
+                        lst[i].invented_name = True
                         self.produced_short_boxes[new_name] = lst[i]
             else:
                 pop = shortop.add_op_to_short(self)
@@ -321,6 +328,10 @@
 empty_info = EmptyInfo()
 
 class ShortPreambleBuilder(object):
+    """ ShortPreambleBuilder is used during optimizing of the peeled loop,
+    starting from short_boxes exported from the preamble. It will build
+    the short preamble and necessary extra label arguments
+    """
     def __init__(self, short_boxes, short_inputargs, exported_infos,
                  optimizer=None):
         for produced_op in short_boxes:
@@ -336,6 +347,7 @@
         self.short = []
         self.used_boxes = []
         self.short_preamble_jump = []
+        self.extra_same_as = []
         self.short_inputargs = short_inputargs
 
     def use_box(self, box, preamble_op, optimizer=None):
@@ -363,6 +375,8 @@
         return preamble_op
 
     def add_preamble_op(self, preamble_op):
+        if preamble_op.invented_name:
+            self.extra_same_as.append(preamble_op.op)
         self.used_boxes.append(preamble_op.op)
         self.short_preamble_jump.append(preamble_op.preamble_op)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1159,8 +1159,7 @@
         i1 = getfield_gc_i(p0, descr=valuedescr)
         i2 = int_sub(i1, 1)
         i3 = int_add(i0, i1)
-        #i4 = same_as_i(i2) # This same_as should be killed by backend
-        jump(i3, i1, i2)
+        jump(i3, i2, i1)
         """
         expected = """
         [i0, i1bis, i1]
@@ -8654,5 +8653,34 @@
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_unroll_two_boxes_used_differently(self):
+        ops = """
+        [p0, i0, i2]
+        i1 = int_add(i0, 1)
+        i3 = int_add(i1, i2)
+        i4 = getfield_gc_i(p0, descr=valuedescr)
+        escape_n(i4)
+        setfield_gc(p0, i1, descr=valuedescr)
+        jump(p0, i0, i3)
+        """
+        preamble = """
+        [p0, i0, i2]
+        i1 = int_add(i0, 1)
+        i3 = int_add(i1, i2)
+        i4 = getfield_gc_i(p0, descr=valuedescr)
+        escape_n(i4)
+        setfield_gc(p0, i1, descr=valuedescr)
+        ii = same_as_i(i1)
+        jump(p0, i0, i3, i1, ii)        
+        """
+        expected = """
+        [p0, i0, i2, i4, i5]
+        i3 = int_add(i4, i2)
+        escape_n(i5)
+        setfield_gc(p0, i4, descr=valuedescr)
+        jump(p0, i0, i3, i4, i4)
+        """
+        self.optimize_loop(ops, expected, preamble)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -474,9 +474,10 @@
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
         start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
-        preamble.operations = [start_label] + preamble_ops
         inputargs = start_state.end_args + loop_info.extra_label_args
         emit_end_label = ResOperation(rop.LABEL, inputargs)
+        preamble.operations = ([start_label] + preamble_ops +
+                               loop_info.extra_same_as + [emit_end_label])
         loop.inputargs = inputargs
         loop.operations = [emit_end_label] + ops
         return Info(preamble, loop_info.short_preamble)
@@ -495,7 +496,8 @@
                       loop.operations
     if not jump:
         assert newloop.operations[-1].getopnum() == rop.JUMP
-        newloop.operations = newloop.operations[:-1]
+        newloop.operations[-1] = newloop.operations[-1].copy_and_change(
+            rop.LABEL)
     return newloop
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -124,7 +124,8 @@
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
         return (UnrollInfo(self.short_preamble_producer.build_short_preamble(),
-                           used_boxes),
+                           used_boxes,
+                           self.short_preamble_producer.extra_same_as),
                 self.optimizer._newoperations)
 
     def filter_extra_jump_args(self, sp, jump_args, extra_jump_args):
@@ -222,10 +223,12 @@
 
     * short_preamble - list of operations that go into short preamble
     * extra_label_args - additional things to put in the label
+    * extra_same_as - list of extra same as to add at the end of the preamble
     """
-    def __init__(self, short_preamble, extra_label_args):
+    def __init__(self, short_preamble, extra_label_args, extra_same_as):
         self.short_preamble = short_preamble
         self.extra_label_args = extra_label_args
+        self.extra_same_as = extra_same_as
             
 class ExportedState(LoopInfo):
     """ Exported state consists of a few pieces of information:

From noreply at buildbot.pypy.org  Thu Aug 13 13:33:39 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 13 Aug 2015 13:33:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: (cfbolz,
	arigo): finally finally support str(cls)
Message-ID: <20150813113339.067D41C14A9@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r78958:0822afc04fdd
Date: 2015-08-13 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/0822afc04fdd/

Log:	(cfbolz, arigo): finally finally support str(cls)

diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py
--- a/rpython/rtyper/rpbc.py
+++ b/rpython/rtyper/rpbc.py
@@ -750,6 +750,10 @@
     def get_ll_eq_function(self):
         return None
 
+    def ll_str(self, ptr):
+        cls = lltype.cast_pointer(CLASSTYPE, ptr)
+        return cls.name
+
 
 def ll_cls_hash(cls):
     if not cls:
diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py
--- a/rpython/rtyper/test/test_rpbc.py
+++ b/rpython/rtyper/test/test_rpbc.py
@@ -1656,6 +1656,20 @@
         res = self.interpret(g, [2])
         assert self.ll_to_string(res) == "ASub"
 
+    def test_str_class(self):
+        class Base(object): pass
+        class ASub(Base): pass
+        def g(n):
+            if n == 1:
+                x = Base()
+            else:
+                x = ASub()
+            return str(x.__class__)
+        res = self.interpret(g, [1])
+        assert self.ll_to_string(res) == "Base"
+        res = self.interpret(g, [2])
+        assert self.ll_to_string(res) == "ASub"
+
     def test_bug_callfamily(self):
         def cb1():
             xxx    # never actually called

From noreply at buildbot.pypy.org  Thu Aug 13 13:34:43 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 13 Aug 2015 13:34:43 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: merge default
Message-ID: <20150813113443.99C231C14A9@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78959:72c1b4d38529
Date: 2015-08-13 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/72c1b4d38529/

Log:	merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -63,3 +63,6 @@
 .. branch: fix-tinylang-goals
 
 Update tinylang goals to match current rpython
+
+.. branch: vmprof-review
+
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -588,7 +588,8 @@
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
-        if self.fields is None:  # if builtin dtype
+        if self.fields is None and not isinstance(self.itemtype, types.VoidType):  
+            # if builtin dtype (but not w_voiddtype)
             return space.w_None
 
         version = space.int_w(space.getitem(w_data, space.wrap(0)))
diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py
--- a/pypy/module/micronumpy/test/test_dtypes.py
+++ b/pypy/module/micronumpy/test/test_dtypes.py
@@ -350,8 +350,8 @@
         assert np.dtype(xyz).name == 'xyz'
         # another obscure API, used in numpy record.py
         # it seems numpy throws away the subclass type and parses the spec
-        a = np.dtype((xyz, [('x', int), ('y', float)]))
-        assert repr(a) == "dtype([('x', '

Author: Armin Rigo 
Branch: 
Changeset: r78960:f90be60bb5e3
Date: 2015-08-13 14:32 +0100
http://bitbucket.org/pypy/pypy/changeset/f90be60bb5e3/

Log:	We still need to be a bit more subtle in the non-translated version
	of this function

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -725,10 +725,16 @@
     result_w = []
     #
     if not we_are_translated():   # fast path before translation
-        for gcref in roots:       # 'roots' is all objects in this case
-            w_obj = callback(gcref)
-            if w_obj is not None:
-                result_w.append(w_obj)
+        seen = set()
+        while roots:
+            gcref = roots.pop()
+            key = (type(gcref), gcref)
+            if key not in seen:
+                seen.add(key)
+                w_obj = callback(gcref)
+                if w_obj is not None:
+                    result_w.append(w_obj)
+                roots.extend(get_rpy_referents(gcref))
         return result_w
     #
     pending = roots[:]

From noreply at buildbot.pypy.org  Thu Aug 13 16:06:02 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:06:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: This test takes forever if the Python
 process is already a bit big. Run it in a subprocess and it's very fast.
Message-ID: <20150813140602.5B74F1C1106@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78961:eb1030491d7b
Date: 2015-08-13 15:06 +0100
http://bitbucket.org/pypy/pypy/changeset/eb1030491d7b/

Log:	This test takes forever if the Python process is already a bit big.
	Run it in a subprocess and it's very fast.

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -728,9 +728,8 @@
         seen = set()
         while roots:
             gcref = roots.pop()
-            key = (type(gcref), gcref)
-            if key not in seen:
-                seen.add(key)
+            if gcref not in seen:
+                seen.add(gcref)
                 w_obj = callback(gcref)
                 if w_obj is not None:
                     result_w.append(w_obj)
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -1,6 +1,7 @@
 import py
 import sys
 from rpython.tool.udir import udir
+from rpython.tool.version import rpythonroot
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
@@ -16,7 +17,7 @@
                                         " x86-64 CPUs for now")
 
 
-    ROOT = py.path.local(__file__).join('..')
+    ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
     SRC = ROOT.join('src')
 
 
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,4 @@
-import time, os
+import time, os, sys
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
 from rpython.translator.c.test.test_genc import compile
@@ -52,7 +52,9 @@
     return main
 
 def test_interpreted():
-    main()
+    # takes forever if the Python process is already big...
+    import subprocess
+    subprocess.check_call([sys.executable, __file__])
 
 def test_compiled():
     fn = compile(main, [], gcpolicy="minimark")
@@ -60,3 +62,8 @@
         os.unlink(PROF_FILE)
     fn()
     assert os.path.exists(PROF_FILE)
+
+if __name__ == '__main__':
+    setup_module(None)
+    res = main()
+    assert res == 0

From noreply at buildbot.pypy.org  Thu Aug 13 16:15:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:15:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: Revert partially 7a82bbfa8b25: for
 tests, it is possible that several
Message-ID: <20150813141522.B052A1C1106@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78962:056e1a36532a
Date: 2015-08-13 15:15 +0100
http://bitbucket.org/pypy/pypy/changeset/056e1a36532a/

Log:	Revert partially 7a82bbfa8b25: for tests, it is possible that
	several meta_interp() calls each see the same jitdriver in the same
	situation, so 'numreds' is already non-None

diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -79,9 +79,6 @@
             assert methname == 'jit_merge_point', (
                 "reds='auto' is supported only for jit drivers which "
                 "calls only jit_merge_point. Found a call to %s" % methname)
-            if jitdriver.numreds is not None:
-                raise AssertionError("there are multiple jit_merge_points "
-                                     "with the same jitdriver")
             #
             # compute the set of live variables across the jit_marker
             alive_v = set()
@@ -99,7 +96,11 @@
                                            v.concretetype is not lltype.Void]
             reds_v = sort_vars(reds_v)
             op.args.extend(reds_v)
-            jitdriver.numreds = len(reds_v)
+            if jitdriver.numreds is None:
+                jitdriver.numreds = len(reds_v)
+            elif jitdriver.numreds != len(reds_v):
+                raise AssertionError("there are multiple jit_merge_points "
+                                     "with the same jitdriver")
 
 def split_before_jit_merge_point(graph, portalblock, portalopindex):
     """Split the block just before the 'jit_merge_point',

From noreply at buildbot.pypy.org  Thu Aug 13 16:16:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:16:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document vmprof-review
Message-ID: <20150813141624.29E451C1106@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78963:72c351cb910f
Date: 2015-08-13 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/72c351cb910f/

Log:	Document vmprof-review

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -66,3 +66,4 @@
 
 .. branch: vmprof-review
 
+Clean up of vmprof, notably to handle correctly multiple threads

From noreply at buildbot.pypy.org  Thu Aug 13 16:18:29 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 13 Aug 2015 16:18:29 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix logical_and.reduce()
Message-ID: <20150813141829.346B11C1106@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r78964:c9342885423a
Date: 2015-08-13 15:15 +0100
http://bitbucket.org/pypy/pypy/changeset/c9342885423a/

Log:	fix logical_and.reduce()

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1166,6 +1166,7 @@
         assert (logical_xor([True, False, True, False], [1, 2, 0, 0])
                 == [False, True, True, False]).all()
         assert (logical_not([True, False]) == [False, True]).all()
+        assert logical_and.reduce([1.,1.]) == True
 
     def test_logn(self):
         import math
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -511,15 +511,15 @@
         W_Ufunc.__init__(self, name, promote_to_largest, promote_to_float, promote_bools,
                          identity, int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
-        self.bool_result = bool_result
         if name == 'logical_and':
             self.done_func = done_if_false
         elif name == 'logical_or':
             self.done_func = done_if_true
         else:
             self.done_func = None
+        self.bool_result = bool_result or (self.done_func is not None)
         self.simple_binary = (
-            allow_complex and allow_bool and not bool_result and not int_only
+            allow_complex and allow_bool and not self.bool_result and not int_only
             and not complex_to_float and not promote_to_float
             and not promote_bools)
 
@@ -630,7 +630,7 @@
                                             r_dtype.is_complex())):
             raise oefmt(space.w_TypeError,
                 "ufunc '%s' not supported for the input types", self.name)
-        if self.bool_result:
+        if self.bool_result and not self.done_func:
             # XXX: should actually pass the arrays
             dtype = find_result_type(space, [], [l_dtype, r_dtype])
             bool_dtype = get_dtype_cache(space).w_booldtype

From noreply at buildbot.pypy.org  Thu Aug 13 16:18:31 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 13 Aug 2015 16:18:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150813141831.4C1BB1C1106@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r78965:667f48c07e2a
Date: 2015-08-13 15:16 +0100
http://bitbucket.org/pypy/pypy/changeset/667f48c07e2a/

Log:	merge heads

diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -79,9 +79,6 @@
             assert methname == 'jit_merge_point', (
                 "reds='auto' is supported only for jit drivers which "
                 "calls only jit_merge_point. Found a call to %s" % methname)
-            if jitdriver.numreds is not None:
-                raise AssertionError("there are multiple jit_merge_points "
-                                     "with the same jitdriver")
             #
             # compute the set of live variables across the jit_marker
             alive_v = set()
@@ -99,7 +96,11 @@
                                            v.concretetype is not lltype.Void]
             reds_v = sort_vars(reds_v)
             op.args.extend(reds_v)
-            jitdriver.numreds = len(reds_v)
+            if jitdriver.numreds is None:
+                jitdriver.numreds = len(reds_v)
+            elif jitdriver.numreds != len(reds_v):
+                raise AssertionError("there are multiple jit_merge_points "
+                                     "with the same jitdriver")
 
 def split_before_jit_merge_point(graph, portalblock, portalopindex):
     """Split the block just before the 'jit_merge_point',

From noreply at buildbot.pypy.org  Thu Aug 13 16:18:33 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Thu, 13 Aug 2015 16:18:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150813141833.7A9A11C1106@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r78966:9d40d0207632
Date: 2015-08-13 15:18 +0100
http://bitbucket.org/pypy/pypy/changeset/9d40d0207632/

Log:	merge heads

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -66,3 +66,4 @@
 
 .. branch: vmprof-review
 
+Clean up of vmprof, notably to handle correctly multiple threads

From noreply at buildbot.pypy.org  Thu Aug 13 16:37:20 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:37:20 +0200 (CEST)
Subject: [pypy-commit] pypy default: Skip skip skip the rvmprof tests on
	32bit
Message-ID: <20150813143720.BEB6A1C1180@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78967:22d130dae1a6
Date: 2015-08-13 15:28 +0100
http://bitbucket.org/pypy/pypy/changeset/22d130dae1a6/

Log:	Skip skip skip the rvmprof tests on 32bit

diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -2,6 +2,7 @@
 from rpython.rlib.rvmprof.rvmprof import _get_vmprof, VMProfError
 from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code, MAX_FUNC_NAME
 from rpython.rlib.rvmprof.rvmprof import _was_registered
+from rpython.rlib.rvmprof.cintf import VMProfPlatformUnsupported
 
 #
 # See README.txt.
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -196,6 +196,11 @@
     (including 'self' if applicable).
     """
     def decorate(func):
+        try:
+            _get_vmprof()
+        except cintf.VMProfPlatformUnsupported:
+            return func
+
         if hasattr(func, 'im_self'):
             assert func.im_self is None
             func = func.im_func
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -8,7 +8,10 @@
 
     class MyCode:
         pass
-    rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    try:
+        rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    except rvmprof.VMProfPlatformUnsupported:
+        pass
 
     @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
     def main(code, num):
@@ -29,7 +32,10 @@
 
     class MyCode:
         pass
-    rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    try:
+        rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    except rvmprof.VMProfPlatformUnsupported:
+        pass
 
     class A:
         pass
@@ -54,7 +60,10 @@
 
     class MyCode:
         pass
-    rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    try:
+        rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    except rvmprof.VMProfPlatformUnsupported, e:
+        py.test.skip(str(e))
 
     @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
     def main(code, num):
@@ -79,7 +88,10 @@
         pass
     def get_name(code):
         return 'py:code:52:x'
-    rvmprof.register_code_object_class(MyCode, get_name)
+    try:
+        rvmprof.register_code_object_class(MyCode, get_name)
+    except rvmprof.VMProfPlatformUnsupported, e:
+        py.test.skip(str(e))
 
     @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
     def main(code, num):
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,4 @@
-import time, os, sys
+import time, os, sys, py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
 from rpython.translator.c.test.test_genc import compile
@@ -14,7 +14,10 @@
 
 
 def setup_module(mod):
-    rvmprof.register_code_object_class(MyCode, MyCode.get_name)
+    try:
+        rvmprof.register_code_object_class(MyCode, MyCode.get_name)
+    except rvmprof.VMProfPlatformUnsupported, e:
+        py.test.skip(str(e))
 
 
 @rvmprof.vmprof_execute_code("interp", lambda code: code)

From noreply at buildbot.pypy.org  Thu Aug 13 16:37:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:37:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: More skipping on 32bit
Message-ID: <20150813143722.DB0AD1C1180@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78968:572a588ad352
Date: 2015-08-13 15:36 +0100
http://bitbucket.org/pypy/pypy/changeset/572a588ad352/

Log:	More skipping on 32bit

diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib.rvmprof import VMProfPlatformUnsupported
 
 class Module(MixedModule):
     """
@@ -19,4 +20,7 @@
 # already found by the annotator to be the original empty
 # method, and the annotator doesn't notice that interp_vmprof.py
 # (loaded later) replaces this method.
-import pypy.module._vmprof.interp_vmprof
+try:
+    import pypy.module._vmprof.interp_vmprof
+except VMProfPlatformUnsupported, e:
+    pass

From noreply at buildbot.pypy.org  Thu Aug 13 16:37:24 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:37:24 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150813143724.EB63F1C1180@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78969:7d34a1f415dc
Date: 2015-08-13 15:37 +0100
http://bitbucket.org/pypy/pypy/changeset/7d34a1f415dc/

Log:	merge heads

diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1166,6 +1166,7 @@
         assert (logical_xor([True, False, True, False], [1, 2, 0, 0])
                 == [False, True, True, False]).all()
         assert (logical_not([True, False]) == [False, True]).all()
+        assert logical_and.reduce([1.,1.]) == True
 
     def test_logn(self):
         import math
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -511,15 +511,15 @@
         W_Ufunc.__init__(self, name, promote_to_largest, promote_to_float, promote_bools,
                          identity, int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
-        self.bool_result = bool_result
         if name == 'logical_and':
             self.done_func = done_if_false
         elif name == 'logical_or':
             self.done_func = done_if_true
         else:
             self.done_func = None
+        self.bool_result = bool_result or (self.done_func is not None)
         self.simple_binary = (
-            allow_complex and allow_bool and not bool_result and not int_only
+            allow_complex and allow_bool and not self.bool_result and not int_only
             and not complex_to_float and not promote_to_float
             and not promote_bools)
 
@@ -630,7 +630,7 @@
                                             r_dtype.is_complex())):
             raise oefmt(space.w_TypeError,
                 "ufunc '%s' not supported for the input types", self.name)
-        if self.bool_result:
+        if self.bool_result and not self.done_func:
             # XXX: should actually pass the arrays
             dtype = find_result_type(space, [], [l_dtype, r_dtype])
             bool_dtype = get_dtype_cache(space).w_booldtype

From noreply at buildbot.pypy.org  Thu Aug 13 16:40:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Aug 2015 16:40:58 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix the same_as on inputargs I
	think
Message-ID: <20150813144058.32A051C0221@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78970:dd3b2a28fc96
Date: 2015-08-13 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/dd3b2a28fc96/

Log:	fix the same_as on inputargs I think

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -60,6 +60,8 @@
             info.produce_short_preamble_ops(structbox, descr, index, optimizer,
                                             shortboxes)
         return
+
+        XXXXX
         for structvalue in self._cached_fields_getfield_op.keys():
             op = self._cached_fields_getfield_op[structvalue]
             if not op:
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,6 +1,6 @@
 
 from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers,\
-     rop, AbstractResOp
+     rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
      TreeLoop
 from rpython.jit.metainterp.optimizeopt import info
@@ -185,12 +185,16 @@
 dummy_short_op = ProducedShortOp(None, None)
 
 
-class ShortInputArg(AbstractProducedShortOp):
-    def __init__(self, preamble_op):
+class ShortInputArg(AbstractShortOp):
+    def __init__(self, res, preamble_op):
+        self.res = res
         self.preamble_op = preamble_op
 
-    def produce_op(self, opt, exported_infos):
-        pass
+    def add_op_to_short(self, sb):
+        return ProducedShortOp(self, self.preamble_op)
+
+    def produce_op(self, opt, preamble_op, exported_infos, invented_name):
+        assert not invented_name
 
     def __repr__(self):
         return "INP(%r)" % (self.preamble_op,)
@@ -213,7 +217,7 @@
         for box in inputargs:
             if box in label_d:
                 renamed = OpHelpers.inputarg_from_tp(box.type)
-                self.produced_short_boxes[box] = ShortInputArg(renamed)
+                self.potential_ops[box] = ShortInputArg(box, renamed)
 
         optimizer.produce_potential_short_preamble_ops(self)
 
@@ -317,7 +321,7 @@
 
     def add_heap_op(self, op, getfield_op):
         # or an inputarg
-        if isinstance(op, Const) or op in self.produced_short_boxes:
+        if isinstance(op, Const):
             self.const_short_boxes.append(HeapOp(op, getfield_op))
             return # we should not be called from anywhere
         self.add_potential_op(op, HeapOp(op, getfield_op))
@@ -352,7 +356,7 @@
 
     def use_box(self, box, preamble_op, optimizer=None):
         for arg in preamble_op.getarglist():
-            if isinstance(arg, Const):
+            if isinstance(arg, Const) or isinstance(arg, AbstractInputArg):
                 pass
             elif arg.get_forwarded() is None:
                 pass
@@ -362,7 +366,6 @@
                 if info is not empty_info:
                     info.make_guards(arg, self.short)
                 arg.set_forwarded(None)
-                #self.force_info_from(arg) <- XXX?
         self.short.append(preamble_op)
         if preamble_op.is_ovf():
             self.short.append(ResOperation(rop.GUARD_NO_OVERFLOW, [], None))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -853,17 +853,17 @@
         p2sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        # i4 = same_as_i(i1) <- investigate
-        jump(i1, p2, p2sub)
-        """
-        expected = """
-        [i1, p2, p10]
-        escape_n(i1)
+        i4 = same_as_i(i1)
+        jump(i1, p2, p2sub, i4)
+        """
+        expected = """
+        [i1, p2, p10, i10]
+        escape_n(i10)
         p1 = new_with_vtable(descr=nodesize)
         p3sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        jump(i1, p1, p3sub)
+        jump(i1, p1, p3sub, i1)
         """
         self.optimize_loop(ops, expected, preamble)
 
@@ -2241,6 +2241,7 @@
         """
         expected = """
         [p1, i1, i2]
+        setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2)
         """
         # in this case, all setfields are removed, because we can prove
@@ -7347,29 +7348,31 @@
         ops = """
         [p0, p1, ii, ii2]
         i1 = getfield_gc_i(p0, descr=valuedescr)
-        i2 = getfield_gc_i(p1, descr=otherdescr)
+        i2 = getfield_gc_i(p1, descr=chardescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
-        setfield_gc(p1, ii, descr=otherdescr)
+        setfield_gc(p1, ii, descr=chardescr)
         i4 = getfield_gc_i(p0, descr=valuedescr)
-        i5 = getfield_gc_i(p1, descr=otherdescr)
+        i5 = getfield_gc_i(p1, descr=chardescr)
         jump(p0, p1, ii2, ii)
         """
         preamble = """
         [p0, p1, ii, ii2]
         i1 = getfield_gc_i(p0, descr=valuedescr)
-        i2 = getfield_gc_i(p1, descr=otherdescr)
+        i2 = getfield_gc_i(p1, descr=chardescr)
         i3 = int_add(i1, i2)
         setfield_gc(p0, ii, descr=valuedescr)
-        setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii, ii, ii)
-        """
-        expected = """
-        [p0, p1, ii, ii2, i1, i2]
-        i3 = int_add(i1, i2)
-        setfield_gc(p0, ii, descr=valuedescr)
-        setfield_gc(p1, ii, descr=otherdescr)
-        jump(p0, p1, ii2, ii, ii, ii)
+        setfield_gc(p1, ii, descr=chardescr)
+        i10 = same_as_i(ii)
+        i11 = same_as_i(ii)
+        jump(p0, p1, ii2, ii, i10, i11)
+        """
+        expected = """
+        [p0, p1, i1, i2, i7, i8]
+        i3 = int_add(i7, i8)
+        setfield_gc(p0, i1, descr=valuedescr)
+        setfield_gc(p1, i1, descr=chardescr)
+        jump(p0, p1, i2, i1, i1, i1)
         """
         self.optimize_loop(ops, expected, preamble)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -474,7 +474,7 @@
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
         start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
-        inputargs = start_state.end_args + loop_info.extra_label_args
+        inputargs = loop_info.label_args[:]
         emit_end_label = ResOperation(rop.LABEL, inputargs)
         preamble.operations = ([start_label] + preamble_ops +
                                loop_info.extra_same_as + [emit_end_label])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -117,29 +117,29 @@
                                     self.optimizer, force_boxes=True)
         extra_jump_args = self.inline_short_preamble(jump_args)
         # remove duplicates, removes stuff from used boxes too
-        extra_jump_args, used_boxes = self.filter_extra_jump_args(
-            self.short_preamble_producer, jump_args,
-            extra_jump_args)
-        jump_args += extra_jump_args
+        label_args, jump_args = self.filter_extra_jump_args(
+            start_label.getarglist() + self.short_preamble_producer.used_boxes,
+            jump_args + extra_jump_args)
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer._newoperations.append(jump_op)
         return (UnrollInfo(self.short_preamble_producer.build_short_preamble(),
-                           used_boxes,
+                           label_args,
                            self.short_preamble_producer.extra_same_as),
                 self.optimizer._newoperations)
 
-    def filter_extra_jump_args(self, sp, jump_args, extra_jump_args):
+    def filter_extra_jump_args(self, label_args, jump_args):
+        new_label_args = []
+        new_jump_args = []
+        assert len(label_args) == len(jump_args)
         d = {}
-        for arg in jump_args:
+        for i in range(len(label_args)):
+            arg = label_args[i]
+            if arg in d:
+                continue
+            new_label_args.append(arg)
+            new_jump_args.append(jump_args[i])
             d[arg] = None
-        new_jump_args = []
-        new_used_boxes = []
-        for i in range(len(extra_jump_args)):
-            if extra_jump_args[i] in d:
-                continue
-            new_jump_args.append(extra_jump_args[i])
-            new_used_boxes.append(sp.used_boxes[i])
-        return new_jump_args, new_used_boxes
+        return new_label_args, new_jump_args
 
     def inline_short_preamble(self, jump_args):
         sb = self.short_preamble_producer
@@ -222,12 +222,12 @@
     """ A state after optimizing the peeled loop, contains the following:
 
     * short_preamble - list of operations that go into short preamble
-    * extra_label_args - additional things to put in the label
+    * label_args - additional things to put in the label
     * extra_same_as - list of extra same as to add at the end of the preamble
     """
-    def __init__(self, short_preamble, extra_label_args, extra_same_as):
+    def __init__(self, short_preamble, label_args, extra_same_as):
         self.short_preamble = short_preamble
-        self.extra_label_args = extra_label_args
+        self.label_args = label_args
         self.extra_same_as = extra_same_as
             
 class ExportedState(LoopInfo):

From noreply at buildbot.pypy.org  Thu Aug 13 16:57:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 16:57:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2113: str.expandtabs(0)
Message-ID: <20150813145707.1B1B01C0221@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78971:0b6868a4c2e2
Date: 2015-08-13 15:57 +0100
http://bitbucket.org/pypy/pypy/changeset/0b6868a4c2e2/

Log:	Issue #2113: str.expandtabs(0)

diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -195,7 +195,8 @@
             splitted = split(value, self._chr('\t'))
 
         try:
-            ovfcheck(len(splitted) * tabsize)
+            if tabsize > 0:
+                ovfcheck(len(splitted) * tabsize)
         except OverflowError:
             raise oefmt(space.w_OverflowError, "new string is too long")
         expanded = oldtoken = splitted.pop(0)
@@ -210,6 +211,8 @@
     def _tabindent(self, token, tabsize):
         """calculates distance behind the token to the next tabstop"""
 
+        if tabsize <= 0:
+            return 0
         distance = tabsize
         if token:
             distance = 0
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -388,6 +388,10 @@
             skip("Wrong platform")
         raises((MemoryError, OverflowError), 't\tt\t'.expandtabs, sys.maxint)
 
+    def test_expandtabs_0(self):
+        assert 'x\ty'.expandtabs(0) == 'xy'
+        assert 'x\ty'.expandtabs(-42) == 'xy'
+
     def test_splitlines(self):
         s = ""
         assert s.splitlines() == []
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -494,6 +494,10 @@
             skip("Wrong platform")
         raises((OverflowError, MemoryError), u't\tt\t'.expandtabs, sys.maxint)
 
+    def test_expandtabs_0(self):
+        assert u'x\ty'.expandtabs(0) == u'xy'
+        assert u'x\ty'.expandtabs(-42) == u'xy'
+
     def test_translate(self):
         assert u'bbbc' == u'abababc'.translate({ord('a'):None})
         assert u'iiic' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i')})

From noreply at buildbot.pypy.org  Thu Aug 13 17:21:51 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Aug 2015 17:21:51 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: (fijal,
	arigo) change subtle order issues
Message-ID: <20150813152151.27EF51C0221@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78972:58f957a4e8fb
Date: 2015-08-13 17:06 +0200
http://bitbucket.org/pypy/pypy/changeset/58f957a4e8fb/

Log:	(fijal, arigo) change subtle order issues

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -161,9 +161,11 @@
                 subbox = optforce.force_box(fld)
                 setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox],
                                           descr=flddescr)
-                optforce._emit_operation(setfieldop)
-                if optforce.optheap is not None:
-                    optforce.optheap.register_dirty_field(flddescr, op, self)
+                self._fields[i] = None
+                optforce.optheap.propagate_forward(setfieldop)
+                #optforce._emit_operation(setfieldop)
+                #if optforce.optheap is not None:
+                #    optforce.optheap.register_dirty_field(flddescr, op, self)
 
     def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         lst = self.vdescr.get_all_fielddescrs()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -823,8 +823,8 @@
         i3 = getfield_gc_i(p2, descr=valuedescr)
         escape_n(i3)
         p4 = new_with_vtable(descr=nodesize)
+        p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p4, i1, descr=valuedescr)
-        p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
@@ -851,8 +851,8 @@
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
         p2sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2, p2sub, descr=nextdescr)
         setfield_gc(p2sub, i1, descr=valuedescr)
-        setfield_gc(p2, p2sub, descr=nextdescr)
         i4 = same_as_i(i1)
         jump(i1, p2, p2sub, i4)
         """
@@ -1217,10 +1217,10 @@
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(descr=nodesize2)
         p2 = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2, p1, descr=nextdescr)
+        setfield_gc(p0, p1, descr=nextdescr)
         setfield_gc(p2, i1, descr=valuedescr)
-        setfield_gc(p2, p1, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        setfield_gc(p0, p1, descr=nextdescr)
         jump(p1)
         """
         loop = """
@@ -1230,10 +1230,10 @@
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(descr=nodesize2)
         p2 = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2, p1, descr=nextdescr)
+        setfield_gc(p0, p1, descr=nextdescr)
         setfield_gc(p2, i1, descr=valuedescr)
-        setfield_gc(p2, p1, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        setfield_gc(p0, p1, descr=nextdescr)
         jump(p1)
         """
         self.optimize_loop(ops, loop, preamble)
@@ -1288,8 +1288,8 @@
         i28 = int_add(i0, 1)
         i29 = int_add(i0, 2)
         p30 = new_with_vtable(descr=nodesize)
+        setfield_gc(p3, p30, descr=nextdescr)
         setfield_gc(p30, i28, descr=valuedescr)
-        setfield_gc(p3, p30, descr=nextdescr)
         #p46 = same_as(p30) # This same_as should be killed by backend
         jump(i29, p30, p3)
         """
@@ -1298,8 +1298,8 @@
         i28 = int_add(i0, 1)
         i29 = int_add(i0, 2)
         p30 = new_with_vtable(descr=nodesize)
+        setfield_gc(p3, p30, descr=nextdescr)
         setfield_gc(p30, i28, descr=valuedescr)
-        setfield_gc(p3, p30, descr=nextdescr)
         jump(i29, p30, p3)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -2706,8 +2706,8 @@
         guard_true(i3) [p1, p4]
         i4 = int_neg(i2)
         p2 = new_with_vtable(descr=nodesize)
+        setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, p4, descr=nextdescr)
-        setfield_gc(p1, p2, descr=nextdescr)
         #i101 = same_as(i4)
         jump(p1, i2, i4, p4) #, i101)
         """
@@ -2715,8 +2715,8 @@
         [p1, i2, i4, p4] #, i5]
         guard_true(i4) [p1, p4]
         p2 = new_with_vtable(descr=nodesize)
+        setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p2, p4, descr=nextdescr)
-        setfield_gc(p1, p2, descr=nextdescr)
         jump(p1, i2, 1, p4) #, i5)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -3953,9 +3953,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p0, p2, descr=nextdescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
@@ -3964,9 +3964,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, 252, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected, expected)
@@ -3993,9 +3993,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p0, p2, descr=nextdescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [p2, i1]
@@ -4004,9 +4004,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         # the point of this test is that 'i1' should show up in the fail_args
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -121,7 +121,7 @@
             start_label.getarglist() + self.short_preamble_producer.used_boxes,
             jump_args + extra_jump_args)
         jump_op = ResOperation(rop.JUMP, jump_args)
-        self.optimizer._newoperations.append(jump_op)
+        self.optimizer.send_extra_operation(jump_op)
         return (UnrollInfo(self.short_preamble_producer.build_short_preamble(),
                            label_args,
                            self.short_preamble_producer.extra_same_as),

From noreply at buildbot.pypy.org  Thu Aug 13 17:21:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Aug 2015 17:21:53 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: (fijal, arigo) fix arrays too
Message-ID: <20150813152153.6989F1C0221@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78973:495132434c50
Date: 2015-08-13 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/495132434c50/

Log:	(fijal, arigo) fix arrays too

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -162,10 +162,10 @@
                 setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox],
                                           descr=flddescr)
                 self._fields[i] = None
-                optforce.optheap.propagate_forward(setfieldop)
-                #optforce._emit_operation(setfieldop)
-                #if optforce.optheap is not None:
-                #    optforce.optheap.register_dirty_field(flddescr, op, self)
+                if optforce.optheap is not None:
+                    optforce.optheap.propagate_forward(setfieldop)
+                else:
+                    optforce.emit_operation(setfieldop)
 
     def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         lst = self.vdescr.get_all_fielddescrs()
@@ -360,10 +360,11 @@
                 setop = ResOperation(rop.SETARRAYITEM_GC,
                                      [op, ConstInt(i), subbox],
                                       descr=arraydescr)
-                optforce._emit_operation(setop)
+                self._items[i] = None
                 if optforce.optheap is not None:
-                    optforce.optheap.register_dirty_array_field(op,
-                        arraydescr, i, self)
+                    optforce.optheap.propagate_forward(setop)
+                else:
+                    optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
     def setitem(self, index, struct_op, op, cf=None, optheap=None):

From noreply at buildbot.pypy.org  Thu Aug 13 17:21:55 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Aug 2015 17:21:55 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: "fix" the test
Message-ID: <20150813152155.7BD051C0221@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78974:851d6c04091a
Date: 2015-08-13 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/851d6c04091a/

Log:	"fix" the test

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -6016,6 +6016,8 @@
         """
         expected = """
         [p0]
+        p1 = getfield_gc_r(p0, descr=nextdescr) # to be killed by the backend
+        setfield_gc(p0, p0, descr=nextdescr)
         jump(p0)
         """
         self.optimize_loop(ops, expected, preamble)

From noreply at buildbot.pypy.org  Thu Aug 13 17:21:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 13 Aug 2015 17:21:57 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: (fijal,
	arigo) fix some more tests
Message-ID: <20150813152157.9D13F1C0221@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78975:c8de5fdbe7ee
Date: 2015-08-13 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/c8de5fdbe7ee/

Log:	(fijal, arigo) fix some more tests

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -490,7 +490,10 @@
 
     def force_lazy_setfields_and_arrayitems_for_guard(self):
         pendingfields = []
-        for descr, cf in self.cached_fields.iteritems():
+        items = self.cached_fields.items()
+        if not we_are_translated():
+            items.sort(key=str, reverse=True)
+        for descr, cf in items:
             op = cf._lazy_setfield
             if op is None:
                 continue
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,4 +1,5 @@
 
+from collections import OrderedDict
 from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers,\
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
@@ -206,7 +207,7 @@
     def create_short_boxes(self, optimizer, inputargs, label_args):
         # all the potential operations that can be produced, subclasses
         # of AbstractShortOp
-        self.potential_ops = {}
+        self.potential_ops = OrderedDict()
         self.produced_short_boxes = {}
         # a way to produce const boxes, e.g. setfield_gc(p0, Const).
         # We need to remember those, but they don't produce any new boxes
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7393,8 +7393,8 @@
         """
         expected = """
         [p0, p1, p3, ii, ii2, i1, i2]
-        setfield_gc(p3, i1, descr=adescr)
-        setfield_gc(p3, i2, descr=abisdescr)
+        setfield_gc(p3, i1, descr=abisdescr)
+        setfield_gc(p3, i2, descr=adescr)
         i5 = int_gt(ii, 42)
         guard_true(i5) []
         jump(p0, p1, p3, ii2, ii, i1, i2)
@@ -7435,7 +7435,7 @@
         guard_nonnull_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull_class(p2, ConstClass(node_vtable)) []
-        jump(p0)
+        jump(p1)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,21 +1,13 @@
-import sys
 
-from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const
+from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
      ShortPreambleBuilder, PreambleOp
-from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization, LoopInfo, MININT, MAXINT
-from rpython.jit.metainterp.optimizeopt.virtualstate import (VirtualStateConstructor,
-        BadVirtualState, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     OpHelpers, AbstractInputArg, GuardResOp, AbstractResOp
-from rpython.jit.metainterp import compile
-from rpython.rlib.debug import debug_print, debug_start, debug_stop
-
-
-
+from rpython.jit.metainterp.optimizeopt.virtualstate import (
+    VirtualStateConstructor)
+from rpython.jit.metainterp.resoperation import rop, ResOperation
 
 class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):

From noreply at buildbot.pypy.org  Thu Aug 13 18:51:32 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 13 Aug 2015 18:51:32 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: (arigo,
 cfbolz): this does not make sense any more
Message-ID: <20150813165132.994851C130B@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78976:93a15b82e4dc
Date: 2015-08-13 18:51 +0200
http://bitbucket.org/pypy/pypy/changeset/93a15b82e4dc/

Log:	(arigo, cfbolz): this does not make sense any more

diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py
--- a/pypy/interpreter/valueprof.py
+++ b/pypy/interpreter/valueprof.py
@@ -28,7 +28,6 @@
     def get_int_val(self, w_obj):
         raise NotImplementedError("abstract base")
 
-    @objectmodel.always_inline
     def see_write(self, w_value):
         status = self._vprof_status
         if status == SEEN_TOO_MUCH:

From noreply at buildbot.pypy.org  Thu Aug 13 18:51:34 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 13 Aug 2015 18:51:34 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: (cfbolz, arigo): fix a segfault
Message-ID: <20150813165134.BAEDB1C130B@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78977:f5eeb7187c3e
Date: 2015-08-13 18:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f5eeb7187c3e/

Log:	(cfbolz, arigo): fix a segfault

diff --git a/pypy/interpreter/test/test_valueprof.py b/pypy/interpreter/test/test_valueprof.py
--- a/pypy/interpreter/test/test_valueprof.py
+++ b/pypy/interpreter/test/test_valueprof.py
@@ -66,7 +66,7 @@
     v = ValueProf()
     assert v._vprof_status == SEEN_NOTHING
     v.see_write(Value())
-    v.see_write(Value())
+    v.see_write(OtherValue())
     assert v._vprof_status == SEEN_TOO_MUCH
 
 
@@ -91,6 +91,8 @@
     assert v._vprof_status == SEEN_TOO_MUCH
 
 def test_known_class():
+    import gc
+
     v = ValueProf()
     value = Value()
     assert v._vprof_status == SEEN_NOTHING
@@ -109,3 +111,16 @@
     assert v._vprof_status == SEEN_CONSTANT_CLASS
     v.see_write(ValueInt(5))
     assert v._vprof_status == SEEN_TOO_MUCH
+
+    v = ValueProf()
+    assert v._vprof_status == SEEN_NOTHING
+    v.see_write(Value())
+    assert v._vprof_status == SEEN_CONSTANT_OBJ
+    gc.collect()
+    gc.collect()
+    gc.collect()
+    v.see_write(Value())
+    assert v._vprof_status == SEEN_CONSTANT_CLASS
+    v.see_write(OtherValue())
+    assert v._vprof_status == SEEN_TOO_MUCH
+
diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py
--- a/pypy/interpreter/valueprof.py
+++ b/pypy/interpreter/valueprof.py
@@ -44,10 +44,12 @@
             else:
                 try:
                     self._vprof_value_wref = ref(w_value)
-                    self._vprof_status = SEEN_CONSTANT_OBJ
                 except TypeError:
                     # for tests, which really use unwrapped ints in a few places
                     self._vprof_status = SEEN_TOO_MUCH
+                else:
+                    self._vprof_const_cls = w_value.__class__
+                    self._vprof_status = SEEN_CONSTANT_OBJ
         elif status == SEEN_CONSTANT_INT:
             if self.is_int(w_value):
                 if self.read_constant_int() != self.get_int_val(w_value):
@@ -70,7 +72,7 @@
             if prev_obj is not w_value:
                 if self._vprof_counter >= 200:
                     print "NO LONGER CONSTANT", self._vprof_msg, 'obj', w_value
-                prev_cls = prev_obj.__class__
+                prev_cls = self.read_constant_cls()
                 if prev_cls is w_value.__class__:
                     self._vprof_const_cls = prev_cls
                     self._vprof_status = SEEN_CONSTANT_CLASS

From noreply at buildbot.pypy.org  Thu Aug 13 22:55:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 13 Aug 2015 22:55:22 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #216: add "defined(__hpux)" for
	alloca.h
Message-ID: <20150813205522.8F9E31C1106@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2243:0d32dd30762c
Date: 2015-08-13 21:55 +0100
http://bitbucket.org/cffi/cffi/changeset/0d32dd30762c/

Log:	Issue #216: add "defined(__hpux)" for alloca.h

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -53,7 +53,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif

From noreply at buildbot.pypy.org  Thu Aug 13 23:12:45 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 13 Aug 2015 23:12:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: test,
	fix for broadcasting when we shouldn't
Message-ID: <20150813211245.3B5CE1C0221@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r78978:e84ee03a93ab
Date: 2015-08-14 00:12 +0300
http://bitbucket.org/pypy/pypy/changeset/e84ee03a93ab/

Log:	test, fix for broadcasting when we shouldn't

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -52,11 +52,16 @@
 
     @jit.unroll_safe
     def setslice(self, space, arr):
-        if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
-            raise oefmt(space.w_ValueError,
-                "could not broadcast input array from shape "
-                "(%s) into shape ()",
-                ','.join([str(x) for x in arr.get_shape()]))
+        if len(arr.get_shape()) >  len(self.get_shape()):
+            # record arrays get one extra dimension
+            if not self.dtype.is_record() or \
+                    len(arr.get_shape()) > len(self.get_shape()) + 1:
+                raise oefmt(space.w_ValueError,
+                    "could not broadcast input array from shape "
+                    "(%s) into shape (%s)",
+                    ','.join([str(x) for x in arr.get_shape()]),
+                    ','.join([str(x) for x in self.get_shape()]),
+                    )
         shape = shape_agreement(space, self.get_shape(), arr)
         impl = arr.implementation
         if impl.storage == self.storage:
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2668,6 +2668,15 @@
         a[0, :: -1] = numpy.array([11, 12])
         assert (a == [[12, 11], [5, 10]]).all()
 
+        a = numpy.zeros((3, 2), int)
+        b = numpy.ones((3, 1), int)
+        exc = raises(ValueError, 'a[:, 1] = b')
+        assert str(exc.value) == "could not broadcast " +\
+                "input array from shape (3,1) into shape (3)"
+        a[:, 1] = b[:,0] > 0.5
+        assert (a == [[0, 1], [0, 1], [0, 1]]).all()
+        
+
     def test_ufunc(self):
         from numpy import array
         a = array([[1, 2], [3, 4], [5, 6]])

From noreply at buildbot.pypy.org  Fri Aug 14 01:49:30 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Fri, 14 Aug 2015 01:49:30 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Implement time.clock_gettime(),
 time.clock_settime(), time.clock_getres() and time.CLOCK_xxx constants.
Message-ID: <20150813234930.B462A1C1106@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r78979:0387064aa761
Date: 2015-08-14 01:49 +0200
http://bitbucket.org/pypy/pypy/changeset/0387064aa761/

Log:	Implement time.clock_gettime(), time.clock_settime(),
	time.clock_getres() and time.CLOCK_xxx constants.

diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -1,5 +1,6 @@
 
 from pypy.interpreter.mixedmodule import MixedModule
+from .interp_time import CLOCK_CONSTANTS, cConfig
 import os
 
 _WIN = os.name == "nt"
@@ -21,8 +22,16 @@
     }
 
     if os.name == "posix":
+        interpleveldefs['clock_gettime'] = 'interp_time.clock_gettime'
+        interpleveldefs['clock_settime'] = 'interp_time.clock_settime'
+        interpleveldefs['clock_getres'] = 'interp_time.clock_getres'
         interpleveldefs['tzset'] = 'interp_time.tzset'
 
+    for constant in CLOCK_CONSTANTS:
+        value = getattr(cConfig, constant)
+        if value is not None:
+            interpleveldefs[constant] = 'space.wrap(interp_time.cConfig.%s)' % constant
+
     appleveldefs = {
         'struct_time': 'app_time.struct_time',
         '__doc__': 'app_time.__doc__',
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -4,8 +4,10 @@
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rtime import c_clock_gettime, TIMESPEC
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+import math
 import os
 import sys
 import time as pytime
@@ -113,6 +115,13 @@
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
 
+CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
+                   'CLOCK_PROCESS_CPUTIME_ID', 'CLOCK_REALTIME',
+                   'CLOCK_THREAD_CPUTIME_ID']
+
+for constant in CLOCK_CONSTANTS:
+    setattr(CConfig, constant, platform.DefinedConstantInteger(constant))
+
 if _POSIX:
     calling_conv = 'c'
     CConfig.timeval = platform.Struct("struct timeval",
@@ -178,6 +187,12 @@
 c_localtime = external('localtime', [rffi.TIME_TP], TM_P,
                        save_err=rffi.RFFI_SAVE_ERRNO)
 if _POSIX:
+    c_clock_settime = external('clock_settime',
+                               [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+                               save_err=rffi.RFFI_SAVE_ERRNO)
+    c_clock_getres = external('clock_getres',
+                              [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
+                              save_err=rffi.RFFI_SAVE_ERRNO)
     c_tzset = external('tzset', [], lltype.Void)
 if _WIN:
     win_eci = ExternalCompilationInfo(
@@ -595,6 +610,39 @@
     return space.wrap(float(tt))
 
 if _POSIX:
+    @unwrap_spec(clk_id='c_int')
+    def clock_gettime(space, clk_id):
+        with lltype.scoped_alloc(TIMESPEC) as timespec:
+            ret = c_clock_gettime(clk_id, timespec)
+            if ret != 0:
+                raise OperationError(space.w_OSError,
+                                     space.wrap(_get_error_msg()))
+            result = (float(rffi.getintfield(timespec, 'c_tv_sec')) +
+                      float(rffi.getintfield(timespec, 'c_tv_nsec')) * 1e-9)
+        return space.wrap(result)
+
+    @unwrap_spec(clk_id='c_int', secs=float)
+    def clock_settime(space, clk_id, secs):
+        with lltype.scoped_alloc(TIMESPEC) as timespec:
+            frac = math.fmod(secs, 1.0)
+            rffi.setintfield(timespec, 'c_tv_sec', int(secs))
+            rffi.setintfield(timespec, 'c_tv_nsec', int(frac * 1e9))
+            ret = c_clock_settime(clk_id, timespec)
+            if ret != 0:
+                raise OperationError(space.w_OSError,
+                                     space.wrap(_get_error_msg()))
+
+    @unwrap_spec(clk_id='c_int')
+    def clock_getres(space, clk_id):
+        with lltype.scoped_alloc(TIMESPEC) as timespec:
+            ret = c_clock_getres(clk_id, timespec)
+            if ret != 0:
+                raise OperationError(space.w_OSError,
+                                     space.wrap(_get_error_msg()))
+            result = (float(rffi.getintfield(timespec, 'c_tv_sec')) +
+                      float(rffi.getintfield(timespec, 'c_tv_nsec')) * 1e-9)
+        return space.wrap(result)
+
     def tzset(space):
         """tzset()
 
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -34,6 +34,28 @@
         t2 = time.time()
         assert t1 != t2       # the resolution should be at least 0.01 secs
 
+    def test_clock_realtime(self):
+        import os
+        if os.name != "posix":
+            skip("clock_gettime available only under Unix")
+        import time
+        t1 = time.clock_gettime(time.CLOCK_REALTIME)
+        assert isinstance(t1, float)
+        time.sleep(time.clock_getres(time.CLOCK_REALTIME))
+        t2 = time.clock_gettime(time.CLOCK_REALTIME)
+        assert t1 != t2
+
+    def test_clock_monotonic(self):
+        import os
+        if os.name != "posix":
+            skip("clock_gettime available only under Unix")
+        import time
+        t1 = time.clock_gettime(time.CLOCK_MONOTONIC)
+        assert isinstance(t1, float)
+        time.sleep(time.clock_getres(time.CLOCK_MONOTONIC))
+        t2 = time.clock_gettime(time.CLOCK_MONOTONIC)
+        assert t1 < t2
+
     def test_ctime(self):
         import time
         raises(TypeError, time.ctime, "foo")

From noreply at buildbot.pypy.org  Fri Aug 14 07:09:53 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 14 Aug 2015 07:09:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: numpy __setitem__ calls subclass's
 __getitem__. Mark as TODO
Message-ID: <20150814050953.931EB1C0726@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r78980:7089cedc9193
Date: 2015-08-14 08:03 +0300
http://bitbucket.org/pypy/pypy/changeset/7089cedc9193/

Log:	numpy __setitem__ calls subclass's __getitem__. Mark as TODO

diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -247,6 +247,20 @@
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
+        # TODO: multiarray/mapping.c calls a subclass's __getitem__ here, which
+        # is a big performance hit but necessary for the matrix class. The original
+        # C code is like:
+        #/*
+        #* WARNING: There is a huge special case here. If this is not a
+        #*          base class array, we have to get the view through its
+        #*          very own index machinery.
+        #*          Many subclasses should probably call __setitem__
+        #*          with a base class ndarray view to avoid this.
+        #*/
+        #else if (!(index_type & (HAS_FANCY | HAS_SCALAR_ARRAY))
+        #        && !PyArray_CheckExact(self)) {
+        #view = (PyArrayObject *)PyObject_GetItem((PyObject *)self, ind);
+
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
             self.setitem_filter(space, w_idx, convert_to_array(space, w_value))

From noreply at buildbot.pypy.org  Fri Aug 14 12:10:24 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 14 Aug 2015 12:10:24 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: (cfbolz,
 arigo around): deal with non-constant classes in record_exact_class
Message-ID: <20150814101024.15DAC1C0473@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78981:d34953857d58
Date: 2015-08-14 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/d34953857d58/

Log:	(cfbolz, arigo around): deal with non-constant classes in
	record_exact_class

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -334,7 +334,9 @@
     def optimize_RECORD_EXACT_CLASS(self, op):
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
-        assert isinstance(expectedclassbox, Const)
+        if not isinstance(expectedclassbox, Const):
+            # can't optimize
+            return
         realclassbox = value.get_constant_class(self.optimizer.cpu)
         if realclassbox is not None:
             assert realclassbox.same_constant(expectedclassbox)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7098,6 +7098,20 @@
         jump(p1)
         """
         self.optimize_loop(ops, expected)
+        ops = """
+        [p0, i0]
+        p1 = getfield_gc(p0, descr=nextdescr)
+        record_exact_class(p1, i0)
+        guard_class(p1, ConstClass(node_vtable)) []
+        jump(p1, i0)
+        """
+        expected = """
+        [p0, i0]
+        p1 = getfield_gc(p0, descr=nextdescr)
+        guard_class(p1, ConstClass(node_vtable)) []
+        jump(p1, i0)
+        """
+        self.optimize_loop(ops, expected)
 
     def test_quasi_immut(self):
         ops = """

From noreply at buildbot.pypy.org  Fri Aug 14 13:11:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Aug 2015 13:11:50 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix two tests
Message-ID: <20150814111150.B3D841C11C8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78982:a12ed2e82b13
Date: 2015-08-13 17:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a12ed2e82b13/

Log:	fix two tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8291,13 +8291,13 @@
         call_n(i2, descr=nonwritedescr)
         setfield_gc(p22, i1, descr=valuedescr)
         guard_nonnull_class(p18, ConstClass(node_vtable)) []
-        i10 = same_as(i1)
+        i10 = same_as_i(i1)
         jump(p22, p18, i1, i10)
         """
         short = """
         [p22, p18, i1]
         i2 = getfield_gc_i(p22, descr=valuedescr)
-        jump(p22, p18, i1, i2)
+        jump(i2)
         """
         expected = """
         [p22, p18, i1, i2]
@@ -8310,18 +8310,18 @@
     def test_cache_setfield_across_loop_boundaries(self):
         ops = """
         [p1]
-        p2 = getfield_gc_r(p1, descr=valuedescr)
+        p2 = getfield_gc_r(p1, descr=nextdescr)
         guard_nonnull_class(p2, ConstClass(node_vtable)) []
         call_n(p2, descr=nonwritedescr)
         p3 = new_with_vtable(descr=nodesize)
-        setfield_gc(p1, p3, descr=valuedescr)
+        setfield_gc(p1, p3, descr=nextdescr)
         jump(p1)
         """
         expected = """
         [p1, p2]
         call_n(p2, descr=nonwritedescr)
         p3 = new_with_vtable(descr=nodesize)
-        setfield_gc(p1, p3, descr=valuedescr)
+        setfield_gc(p1, p3, descr=nextdescr)
         jump(p1, p3)
         """
         self.optimize_loop(ops, expected)

From noreply at buildbot.pypy.org  Fri Aug 14 13:11:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 14 Aug 2015 13:11:52 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: first version of compile_trace
Message-ID: <20150814111152.E8EA01C11C8@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r78983:31806f1065f3
Date: 2015-08-14 13:12 +0200
http://bitbucket.org/pypy/pypy/changeset/31806f1065f3/

Log:	first version of compile_trace

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -61,19 +61,20 @@
     """ This represents label() ops jump with no extra info associated with
     the label
     """
-    def __init__(self, start_label, operations, call_pure_results=None):
+    def __init__(self, start_label, operations, call_pure_results=None,
+                 enable_opts=None):
         self.start_label = start_label
         self.operations = operations
         self.call_pure_results = call_pure_results
+        self.enable_opts = enable_opts
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer
 
-        assert not unroll
+        #assert not unroll
         opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.propagate_all_forward(self.start_label.getarglist(),
-                                         self.operations,
-                                         self.call_pure_results)
+            self.operations, self.call_pure_results, self.enable_opts)
 
 class UnrolledLoopData(CompileData):
     """ This represents label() ops jump with extra info that's from the
@@ -973,7 +974,7 @@
         metainterp_sd = metainterp.staticdata
         jitdriver_sd = metainterp.jitdriver_sd
         new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
-        propagate_original_jitcell_token(new_loop)
+        #propagate_original_jitcell_token(new_loop)
         send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
                              metainterp_sd, new_loop, "entry bridge")
         # send the new_loop to warmspot.py, to be called directly the next time
@@ -994,30 +995,48 @@
     #
     # Attempt to use optimize_bridge().  This may return None in case
     # it does not work -- i.e. none of the existing old_loop_tokens match.
-    new_trace = create_empty_loop(metainterp)
-    new_trace.inputargs = metainterp.history.inputargs[:]
+    #new_trace = create_empty_loop(metainterp)
+    #new_trace.inputargs = metainterp.history.inputargs[:]
 
-    new_trace.operations = metainterp.history.operations[:]
+    #new_trace.operations = metainterp.history.operations[:]
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     state = jitdriver_sd.warmstate
-    if isinstance(resumekey, ResumeAtPositionDescr):
-        inline_short_preamble = False
-    else:
-        inline_short_preamble = True
+    #if isinstance(resumekey, ResumeAtPositionDescr):
+    #    inline_short_preamble = False
+    #else:
+    #    xxx
+    #    inline_short_preamble = True
+    inputargs = metainterp.history.inputargs[:]
+    operations = metainterp.history.operations
+    label = ResOperation(rop.LABEL, inputargs)
+    jitdriver_sd = metainterp.jitdriver_sd
+    enable_opts = jitdriver_sd.warmstate.enable_opts
+
+    call_pure_results = metainterp.call_pure_results
+
+    data = SimpleCompileData(label, operations,
+                             call_pure_results=call_pure_results,
+                             enable_opts=enable_opts)
     try:
-        state = optimize_trace(metainterp_sd, jitdriver_sd, new_trace,
-                               state.enable_opts,
-                               inline_short_preamble, export_state=True)
+        info, newops = optimize_trace(metainterp_sd, jitdriver_sd, data)
     except InvalidLoop:
-        forget_optimization_info(new_trace.operations)
-        forget_optimization_info(new_trace.inputargs)
+        forget_optimization_info(inputargs)
+        forget_optimization_info(operations)
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
         # InvalidLoop
         debug_print('InvalidLoop in compile_new_bridge')
         return None
 
+    new_trace = create_empty_loop(metainterp)
+    new_trace.inputargs = info.inputargs
+    new_trace.operations = newops
+    resumekey.compile_and_attach(metainterp, new_trace)
+    record_loop_or_bridge(metainterp_sd, new_trace)
+    target_token = new_trace.operations[-1].getdescr()
+    return target_token
+    xxxx
     if new_trace.operations[-1].getopnum() != rop.LABEL:
         # We managed to create a bridge.  Dispatch to resumekey to
         # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7994,19 +7994,19 @@
         ops = """
         [p5, p6]
         i10 = getfield_gc_i(p5, descr=valuedescr)
-        i11 = getfield_gc_i(p6, descr=nextdescr)
+        i11 = getfield_gc_i(p6, descr=chardescr)
         i12 = int_add(i10, 7)
         i13 = int_add(i11, 7)
         call_n(i12, i13, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)
+        setfield_gc(p6, i10, descr=chardescr)
         jump(p5, p6)
         """
         expected = """
-        [p5, p6, i14, i12, i10]
+        [p5, p6, i14, i12]
         i13 = int_add(i14, 7)
         call_n(i12, i13, descr=nonwritedescr)
-        setfield_gc(p6, i10, descr=nextdescr)
-        jump(p5, p6, i10, i12, i10)
+        setfield_gc(p6, i14, descr=chardescr)
+        jump(p5, p6, i14, i12)
         """
         self.optimize_loop(ops, expected)
 

From noreply at buildbot.pypy.org  Fri Aug 14 13:24:15 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 14 Aug 2015 13:24:15 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-opt: new branch to enhance some
 places of the optimization
Message-ID: <20150814112415.258501C133C@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-opt
Changeset: r78984:a82264bddcac
Date: 2015-08-12 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/a82264bddcac/

Log:	new branch to enhance some places of the optimization

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -655,10 +655,6 @@
     def get_input_type_given(self, output_type, op):
         raise AssertionError("cannot infer input type from output type")
 
-    # OLD
-    def determine_output_type(self, op):
-        return None
-
 GUARD_TF = PassThroughOp((PT_INT_GENERIC,))
 INT_OP_TO_VOP = OpToVectorOp((PT_INT_GENERIC, PT_INT_GENERIC), INT_RES)
 FLOAT_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC, PT_FLOAT_GENERIC), FLOAT_RES)
@@ -666,7 +662,7 @@
 LOAD_TRANS = LoadToVectorLoad()
 STORE_TRANS = StoreToVectorStore()
 
-# note that the following definition is x86 arch specific
+# note that the following definition is x86 arch specific (e.g. look at signext)
 ROP_ARG_RES_VECTOR = {
     rop.VEC_INT_ADD:     INT_OP_TO_VOP,
     rop.VEC_INT_SUB:     INT_OP_TO_VOP,
@@ -755,7 +751,7 @@
         renamer = scheduler.renamer
         if candidate.pack:
             for node in candidate.pack.operations:
-                self.unpack_from_vector(candidate.getoperation(), renamer)
+                #self.unpack_from_vector(candidate.getoperation(), renamer)
                 scheduler.scheduled(node)
                 #renamer.rename(node.getoperation())
             self.as_vector_operation(scheduler, candidate.pack)

From noreply at buildbot.pypy.org  Fri Aug 14 13:24:17 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 14 Aug 2015 13:24:17 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-iterator-sharing: started to
 transform call2 to share iterators in the loop,
 works but needs check if the jit codes improve as well
Message-ID: <20150814112417.5770B1C133C@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-iterator-sharing
Changeset: r78985:3cc178b04857
Date: 2015-08-14 13:24 +0200
http://bitbucket.org/pypy/pypy/changeset/3cc178b04857/

Log:	started to transform call2 to share iterators in the loop, works but
	needs check if the jit codes improve as well

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -83,6 +83,11 @@
         self._indices = indices
         self.offset = offset
 
+    def same(self, other):
+        if self.offset == other.offset:
+            return self.iterator.same_shape(other.iterator)
+        return False
+
 
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
@@ -100,6 +105,7 @@
         self.array = array
         self.size = size
         self.ndim_m1 = len(shape) - 1
+        #
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
@@ -113,6 +119,17 @@
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
 
+    def same_shape(self, other):
+        """ if two iterators share the same shape,
+        next() only needs to be called on one!
+        """
+        return (self.contiguous == other.contiguous and
+                self.array.dtype is self.array.dtype and
+                self.shape_m1 == other.shape_m1 and
+                self.strides == other.strides and
+                self.backstrides == other.backstrides and
+                self.factors == other.factors)
+
     @jit.unroll_safe
     def reset(self, state=None, mutate=False):
         index = 0
@@ -196,7 +213,7 @@
         return state.index >= self.size
 
     def getitem(self, state):
-        assert state.iterator is self
+        # assert state.iterator is self
         return self.array.getitem(state.offset)
 
     def getitem_bool(self, state):
@@ -207,7 +224,6 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
-
 def AxisIter(array, shape, axis):
     strides = array.get_strides()
     backstrides = array.get_backstrides()
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -15,7 +15,7 @@
 
 call2_driver = jit.JitDriver(
     name='numpy_call2',
-    greens=['shapelen', 'func', 'left', 'right', 'calc_dtype', 'res_dtype'],
+    greens=['shapelen','state_count', 'left_index', 'right_index', 'left', 'right', 'func', 'calc_dtype', 'res_dtype'],
     reds='auto', vectorize=True)
 
 def call2(space, shape, func, calc_dtype, w_lhs, w_rhs, out):
@@ -38,20 +38,62 @@
     out_iter, out_state = out.create_iter(shape)
     shapelen = len(shape)
     res_dtype = out.get_dtype()
-    while not out_iter.done(out_state):
-        call2_driver.jit_merge_point(shapelen=shapelen, func=func,
+
+    states = [out_state,left_state,right_state]
+    out_index = 0
+    left_index = 1
+    right_index = 2
+    # left == right == out
+    # left == right
+    # left == out
+    # right == out
+    if not right_iter:
+        del states[2]
+    else:
+        if out_state.same(right_state):
+            # (1) out and right are the same -> remove right
+            right_index = 0
+            del states[2]
+    if not left_iter:
+        del states[1]
+        if right_index == 2:
+            right_index = 1
+    else:
+        if out_state.same(left_state):
+            # (2) out and left are the same -> remove left
+            left_index = 0
+            del states[1]
+            if right_index == 2:
+                right_index = 1
+        else:
+            if len(states) == 3: # did not enter (1)
+                if right_iter and right_state.same(left_state):
+                    right_index = 1
+                    del states[2]
+    state_count = len(states)
+    #
+    while not out_iter.done(states[0]):
+        call2_driver.jit_merge_point(shapelen=shapelen,
+                                     func=func,
                                      left=left_iter is None,
                                      right=right_iter is None,
-                                     calc_dtype=calc_dtype, res_dtype=res_dtype)
+                                     state_count=state_count,
+                                     left_index=left_index,
+                                     right_index=right_index,
+                                     calc_dtype=calc_dtype,
+                                     res_dtype=res_dtype)
         if left_iter:
+            left_state = states[left_index]
             w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-            left_state = left_iter.next(left_state)
         if right_iter:
+            right_state = states[right_index]
             w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
-            right_state = right_iter.next(right_state)
         w_out = func(calc_dtype, w_left, w_right)
-        out_iter.setitem(out_state, w_out.convert_to(space, res_dtype))
-        out_state = out_iter.next(out_state)
+        out_iter.setitem(states[0], w_out.convert_to(space, res_dtype))
+        #
+        for i,state in enumerate(states):
+            states[i] = state.iterator.next(state)
+
         # if not set to None, the values will be loop carried
         # (for the var,var case), forcing the vectorization to unpack
         # the vector registers at the end of the loop

From noreply at buildbot.pypy.org  Fri Aug 14 14:56:06 2015
From: noreply at buildbot.pypy.org (rlamy)
Date: Fri, 14 Aug 2015 14:56:06 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix jit tests for numpy logical
	functions
Message-ID: <20150814125606.407B61C0F0D@cobra.cs.uni-duesseldorf.de>

Author: Ronan Lamy 
Branch: 
Changeset: r78986:a79016ff2781
Date: 2015-08-14 13:56 +0100
http://bitbucket.org/pypy/pypy/changeset/a79016ff2781/

Log:	fix jit tests for numpy logical functions

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -28,9 +28,9 @@
             f16 = raw_load(i9, i5, descr=)
             guard_true(i15, descr=...)
             guard_not_invalidated(descr=...)
-            i17 = cast_float_to_int(f16)
-            i19 = int_is_true(i17)
-            guard_true(i19, descr=...)
+            i18 = float_ne(f16, 0.000000)
+            guard_true(i18, descr=...)
+            guard_nonnull_class(p2, ConstClass(W_BoolBox), descr=...)
             i20 = getfield_gc_pure(p2, descr=)
             i21 = int_is_true(i20)
             guard_false(i21, descr=...)
@@ -70,10 +70,10 @@
         assert loop.match("""
             f31 = raw_load(i9, i29, descr=)
             guard_not_invalidated(descr=...)
+            i32 = float_ne(f31, 0.000000)
+            guard_true(i32, descr=...)
             i34 = getarrayitem_raw(#, #, descr=)  # XXX what are these?
             guard_value(i34, #, descr=...)                  # XXX don't appear in
-            i32 = float_ne(f31, 0.000000)
-            guard_true(i32, descr=...)
             i35 = getarrayitem_raw(#, #, descr=)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, i28)

From noreply at buildbot.pypy.org  Fri Aug 14 17:31:13 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 14 Aug 2015 17:31:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: (cfbolz,
 fijal around): don't erase most of the attributes in the standard
Message-ID: <20150814153113.A52771C0F0D@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r78987:485dd1f685d2
Date: 2015-08-14 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/485dd1f685d2/

Log:	(cfbolz, fijal around): don't erase most of the attributes in the
	standard instance classes, to allow the JIT to optimize better (and
	not have to emit mark_opaque_ptr everywhere

diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -539,17 +539,19 @@
     rangen = unroll.unrolling_iterable(range(n))
     nmin1 = n - 1
     rangenmin1 = unroll.unrolling_iterable(range(nmin1))
+    valnmin1 = "_value%s" % nmin1
     class subcls(BaseMapdictObject, supercls):
         def _init_empty(self, map):
-            for i in rangen:
-                setattr(self, "_value%s" % i, erase_item(None))
+            for i in rangenmin1:
+                setattr(self, "_value%s" % i, None)
+            setattr(self, valnmin1, erase_item(None))
             self.map = map
 
         def _has_storage_list(self):
             return self.map.length() > n
 
         def _mapdict_get_storage_list(self):
-            erased = getattr(self, "_value%s" % nmin1)
+            erased = getattr(self, valnmin1)
             return unerase_list(erased)
 
         def _mapdict_read_storage(self, storageindex):
@@ -557,23 +559,21 @@
             if storageindex < nmin1:
                 for i in rangenmin1:
                     if storageindex == i:
-                        erased = getattr(self, "_value%s" % i)
-                        return unerase_item(erased)
+                        return getattr(self, "_value%s" % i)
             if self._has_storage_list():
                 return self._mapdict_get_storage_list()[storageindex - nmin1]
             erased = getattr(self, "_value%s" % nmin1)
             return unerase_item(erased)
 
         def _mapdict_write_storage(self, storageindex, value):
-            erased = erase_item(value)
             for i in rangenmin1:
                 if storageindex == i:
-                    setattr(self, "_value%s" % i, erased)
+                    setattr(self, "_value%s" % i, value)
                     return
             if self._has_storage_list():
                 self._mapdict_get_storage_list()[storageindex - nmin1] = value
                 return
-            setattr(self, "_value%s" % nmin1, erased)
+            setattr(self, "_value%s" % nmin1, erase_item(value))
 
         def _mapdict_storage_length(self):
             if self._has_storage_list():
@@ -585,9 +585,9 @@
             len_storage = len(storage)
             for i in rangenmin1:
                 if i < len_storage:
-                    erased = erase_item(storage[i])
+                    erased = storage[i]
                 else:
-                    erased = erase_item(None)
+                    erased = None
                 setattr(self, "_value%s" % i, erased)
             has_storage_list = self._has_storage_list()
             if len_storage < n:
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -451,12 +451,12 @@
         obj = objectcls()
         obj.user_setup(space, cls)
         obj.setdictvalue(space, "a", w1)
-        assert unerase_item(obj._value0) is w1
+        assert obj._value0 is w1
         assert obj.getdictvalue(space, "a") is w1
         assert obj.getdictvalue(space, "b") is None
         assert obj.getdictvalue(space, "c") is None
         obj.setdictvalue(space, "a", w2)
-        assert unerase_item(obj._value0) is w2
+        assert obj._value0 is w2
         assert obj.getdictvalue(space, "a") == w2
         assert obj.getdictvalue(space, "b") is None
         assert obj.getdictvalue(space, "c") is None
@@ -474,7 +474,7 @@
 
         res = obj.deldictvalue(space, "a")
         assert res
-        assert unerase_item(obj._value0) is w4
+        assert obj._value0 is w4
         assert obj.getdictvalue(space, "a") is None
         assert obj.getdictvalue(space, "b") is w4
         assert obj.getdictvalue(space, "c") is None

From noreply at buildbot.pypy.org  Fri Aug 14 18:50:29 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 14 Aug 2015 18:50:29 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-iterator-sharing: (plan_rich,
 ronan) first working version that generates all five possible call2
 combinations that shares the iterators
Message-ID: <20150814165029.8620D1C11C8@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-iterator-sharing
Changeset: r78988:690ba1eaa6a8
Date: 2015-08-14 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/690ba1eaa6a8/

Log:	(plan_rich, ronan) first working version that generates all five
	possible call2 combinations that shares the iterators

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -88,7 +88,6 @@
             return self.iterator.same_shape(other.iterator)
         return False
 
-
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
                           'strides[*]', 'backstrides[*]', 'factors[*]',
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -2,6 +2,7 @@
 operations. This is the place to look for all the computations that iterate
 over all the array elements.
 """
+import py
 from pypy.interpreter.error import OperationError
 from rpython.rlib import jit
 from rpython.rlib.rstring import StringBuilder
@@ -13,11 +14,6 @@
 from pypy.interpreter.argument import Arguments
 
 
-call2_driver = jit.JitDriver(
-    name='numpy_call2',
-    greens=['shapelen','state_count', 'left_index', 'right_index', 'left', 'right', 'func', 'calc_dtype', 'res_dtype'],
-    reds='auto', vectorize=True)
-
 def call2(space, shape, func, calc_dtype, w_lhs, w_rhs, out):
     if w_lhs.get_size() == 1:
         w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
@@ -40,68 +36,102 @@
     res_dtype = out.get_dtype()
 
     states = [out_state,left_state,right_state]
-    out_index = 0
     left_index = 1
     right_index = 2
     # left == right == out
     # left == right
     # left == out
     # right == out
+    params = (space, shapelen, func, calc_dtype, res_dtype, out,
+              w_left, w_right, left_iter, right_iter, out_iter,
+              left_state, right_state, out_state)
     if not right_iter:
+        # rhs is a scalar
         del states[2]
     else:
+        # rhs is NOT a scalar
         if out_state.same(right_state):
             # (1) out and right are the same -> remove right
             right_index = 0
             del states[2]
+    #
     if not left_iter:
+        # lhs is a scalar
         del states[1]
         if right_index == 2:
             right_index = 1
+            return call2_advance_out_right(*params)
     else:
+        # lhs is NOT a scalar
         if out_state.same(left_state):
             # (2) out and left are the same -> remove left
             left_index = 0
             del states[1]
             if right_index == 2:
                 right_index = 1
+            return call2_advance_out_right(*params)
         else:
             if len(states) == 3: # did not enter (1)
                 if right_iter and right_state.same(left_state):
                     right_index = 1
                     del states[2]
+                    return call2_advance_out_left_eq_right(*params)
+                else:
+                    # worst case
+                    return call2_advance_out_left_right(*params)
+            else:
+                return call2_advance_out_left(*params)
+
     state_count = len(states)
+    if state_count == 1:
+        return call2_advance_out(*params)
+
+    assert 0, "logical problem with the selection of the call 2 case"
+
+def generate_call2_cases(name, left_state, right_state):
+    call2_driver = jit.JitDriver(name='numpy_call2_' + name,
+        greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
+        reds='auto', vectorize=True)
     #
-    while not out_iter.done(states[0]):
-        call2_driver.jit_merge_point(shapelen=shapelen,
-                                     func=func,
-                                     left=left_iter is None,
-                                     right=right_iter is None,
-                                     state_count=state_count,
-                                     left_index=left_index,
-                                     right_index=right_index,
-                                     calc_dtype=calc_dtype,
-                                     res_dtype=res_dtype)
-        if left_iter:
-            left_state = states[left_index]
-            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-        if right_iter:
-            right_state = states[right_index]
-            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
-        w_out = func(calc_dtype, w_left, w_right)
-        out_iter.setitem(states[0], w_out.convert_to(space, res_dtype))
-        #
-        for i,state in enumerate(states):
-            states[i] = state.iterator.next(state)
+    advance_left_state = left_state == "left_state"
+    advance_right_state = right_state == "right_state"
+    code = """
+    def method(space, shapelen, func, calc_dtype, res_dtype, out,
+               w_left, w_right, left_iter, right_iter, out_iter,
+               left_state, right_state, out_state):
+        while not out_iter.done(out_state):
+            call2_driver.jit_merge_point(shapelen=shapelen, func=func,
+                    calc_dtype=calc_dtype, res_dtype=res_dtype)
+            if left_iter:
+                w_left = left_iter.getitem({left_state}).convert_to(space, calc_dtype)
+            if right_iter:
+                w_right = right_iter.getitem({right_state}).convert_to(space, calc_dtype)
+            w_out = func(calc_dtype, w_left, w_right)
+            out_iter.setitem(out_state, w_out.convert_to(space, res_dtype))
+            out_state = out_iter.next(out_state)
+            if advance_left_state and left_iter:
+                left_state = left_iter.next(left_state)
+            if advance_right_state and right_iter:
+                right_state = right_iter.next(right_state)
+            #
+            # if not set to None, the values will be loop carried
+            # (for the var,var case), forcing the vectorization to unpack
+            # the vector registers at the end of the loop
+            if left_iter:
+                w_left = None
+            if right_iter:
+                w_right = None
+        return out
+    """
+    exec(py.code.Source(code.format(left_state=left_state,right_state=right_state)).compile(), locals())
+    method.__name__ = "call2_" + name
+    return method
 
-        # if not set to None, the values will be loop carried
-        # (for the var,var case), forcing the vectorization to unpack
-        # the vector registers at the end of the loop
-        if left_iter:
-            w_left = None
-        if right_iter:
-            w_right = None
-    return out
+call2_advance_out = generate_call2_cases("inc_out", "out_state", "out_state")
+call2_advance_out_left = generate_call2_cases("inc_out_left", "left_state", "out_state")
+call2_advance_out_right = generate_call2_cases("inc_out_right", "out_state", "right_state")
+call2_advance_out_left_eq_right = generate_call2_cases("inc_out_left_eq_right", "left_state", "left_state")
+call2_advance_out_left_right = generate_call2_cases("inc_out_left_right", "left_state", "right_state")
 
 call1_driver = jit.JitDriver(
     name='numpy_call1',
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -911,8 +911,10 @@
     def test_multidim_slice(self):
         result = self.run('multidim_slice')
         assert result == 12
-        self.check_trace_count(2)
-        self.check_vectorized(1,0) # TODO?
+        self.check_trace_count(3)
+        # ::2 creates a view object -> needs an inner loop
+        # that iterates continous chunks of the matrix
+        self.check_vectorized(1,1) 
 
     # NOT WORKING
 

From noreply at buildbot.pypy.org  Fri Aug 14 19:11:31 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Fri, 14 Aug 2015 19:11:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: optimize nullity in the heapcache
Message-ID: <20150814171131.9DA931C0473@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r78989:e226d71c9665
Date: 2015-08-14 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/e226d71c9665/

Log:	optimize nullity in the heapcache

diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -9,6 +9,7 @@
 
     def reset_keep_likely_virtual(self):
         self.known_class = False
+        self.known_nullity = False
         # did we see the allocation during tracing?
         self.seen_allocation = False
         self.is_unescaped = False
@@ -290,6 +291,15 @@
     def class_now_known(self, box):
         self.getvalue(box).known_class = True
 
+    def is_nullity_known(self, box):
+        value = self.getvalue(box, create=False)
+        if value:
+            return value.known_nullity
+        return False
+
+    def nullity_now_known(self, box):
+        self.getvalue(box).known_nullity = True
+
     def is_nonstandard_virtualizable(self, box):
         value = self.getvalue(box, create=False)
         if value:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -369,7 +369,10 @@
         ).compile()
 
     def _establish_nullity(self, box, orgpc):
+        heapcache = self.metainterp.heapcache
         value = box.nonnull()
+        if heapcache.is_nullity_known(box):
+            return value
         if value:
             if not self.metainterp.heapcache.is_class_known(box):
                 self.metainterp.generate_guard(rop.GUARD_NONNULL, box,
@@ -380,6 +383,7 @@
                                                resumepc=orgpc)
                 promoted_box = box.constbox()
                 self.metainterp.replace_box(box, promoted_box)
+        heapcache.nullity_now_known(box)
         return value
 
     @arguments("box", "label", "orgpc")
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -69,6 +69,19 @@
         assert not h.is_class_known(1)
         assert not h.is_class_known(2)
 
+    def test_known_nullity(self):
+        h = HeapCache()
+        assert not h.is_nullity_known(1)
+        assert not h.is_nullity_known(2)
+        h.nullity_now_known(1)
+        assert h.is_nullity_known(1)
+        assert not h.is_nullity_known(2)
+
+        h.reset()
+        assert not h.is_nullity_known(1)
+        assert not h.is_nullity_known(2)
+
+
     def test_nonstandard_virtualizable(self):
         h = HeapCache()
         assert not h.is_nonstandard_virtualizable(1)
diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py
--- a/rpython/jit/metainterp/test/test_tracingopts.py
+++ b/rpython/jit/metainterp/test/test_tracingopts.py
@@ -107,6 +107,28 @@
         assert res == -7 * 2
         self.check_operations_history(getfield_gc=1)
 
+    def test_heap_caching_nonnull(self):
+        class A:
+            def __init__(self, x=None):
+                self.next = x
+        a0 = A()
+        a1 = A()
+        a2 = A(a1)
+        def fn(n):
+            if n > 0:
+                a = a1
+            else:
+                a = a2
+            if a.next:
+                a = A(a.next)
+                result = a.next is not None
+                a0.next = a
+                return result
+            return False
+        res = self.interp_operations(fn, [-7])
+        assert res == True
+        self.check_operations_history(guard_nonnull=1)
+
     def test_heap_caching_while_tracing_invalidation(self):
         class A:
             pass

From noreply at buildbot.pypy.org  Fri Aug 14 22:44:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 14 Aug 2015 22:44:07 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2114: get a new globals
 dictionary for every call to pypy_execute_source_ptr. Expand the
 documentation to point out the drawback of calling pypy_execute_source_ptr
 many times.
Message-ID: <20150814204407.D14641C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78990:20f74886a28e
Date: 2015-08-14 21:46 +0100
http://bitbucket.org/pypy/pypy/changeset/20f74886a28e/

Log:	Issue #2114: get a new globals dictionary for every call to
	pypy_execute_source_ptr. Expand the documentation to point out the
	drawback of calling pypy_execute_source_ptr many times.

diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -46,7 +46,11 @@
    source. It'll acquire the GIL.
 
    Note: this is meant to be called *only once* or a few times at most.  See
-   the `more complete example`_ below.
+   the `more complete example`_ below.  In PyPy <= 2.6.0, the globals
+   dictionary is *reused* across multiple calls, giving potentially
+   strange results (e.g. objects dying too early).  In PyPy >= 2.6.1,
+   you get a new globals dictionary for every call (but then, all globals
+   dictionaries are all kept alive forever, in ``sys._pypy_execute_source``).
 
 .. function:: int pypy_execute_source_ptr(char* source, void* ptr);
 
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -128,13 +128,7 @@
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
-        after = rffi.aroundstate.after
-        if after: after()
-        source = rffi.charp2str(ll_source)
-        res = _pypy_execute_source(source)
-        before = rffi.aroundstate.before
-        if before: before()
-        return rffi.cast(rffi.INT, res)
+        return pypy_execute_source_ptr(ll_source, 0)
 
     @entrypoint('main', [rffi.CCHARP, lltype.Signed],
                 c_name='pypy_execute_source_ptr')
@@ -142,9 +136,7 @@
         after = rffi.aroundstate.after
         if after: after()
         source = rffi.charp2str(ll_source)
-        space.setitem(w_globals, space.wrap('c_argument'),
-                      space.wrap(ll_ptr))
-        res = _pypy_execute_source(source)
+        res = _pypy_execute_source(source, ll_ptr)
         before = rffi.aroundstate.before
         if before: before()
         return rffi.cast(rffi.INT, res)
@@ -169,15 +161,21 @@
         before = rffi.aroundstate.before
         if before: before()
 
-    w_globals = space.newdict()
-    space.setitem(w_globals, space.wrap('__builtins__'),
-                  space.builtin_modules['__builtin__'])
-
-    def _pypy_execute_source(source):
+    def _pypy_execute_source(source, c_argument):
         try:
-            compiler = space.createcompiler()
-            stmt = compiler.compile(source, 'c callback', 'exec', 0)
-            stmt.exec_code(space, w_globals, w_globals)
+            w_globals = space.newdict(module=True)
+            space.setitem(w_globals, space.wrap('__builtins__'),
+                          space.builtin_modules['__builtin__'])
+            space.setitem(w_globals, space.wrap('c_argument'),
+                          space.wrap(c_argument))
+            space.appexec([space.wrap(source), w_globals], """(src, glob):
+                import sys
+                stmt = compile(src, 'c callback', 'exec')
+                if not hasattr(sys, '_pypy_execute_source'):
+                    sys._pypy_execute_source = []
+                sys._pypy_execute_source.append(glob)
+                exec stmt in glob
+            """)
         except OperationError, e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))

From noreply at buildbot.pypy.org  Sat Aug 15 00:16:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 15 Aug 2015 00:16:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2112: [patch by david n.]
 properly detect arm on FreeBSD
Message-ID: <20150814221631.02BFF1C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r78991:9106028eee0f
Date: 2015-08-15 00:16 +0200
http://bitbucket.org/pypy/pypy/changeset/9106028eee0f/

Log:	Issue #2112: [patch by david n.] properly detect arm on FreeBSD

diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -63,6 +63,7 @@
             'AMD64': MODEL_X86,    # win64
             'armv7l': MODEL_ARM,
             'armv6l': MODEL_ARM,
+            'arm': MODEL_ARM,      # freebsd
             }.get(mach)
 
     if result is None:

From noreply at buildbot.pypy.org  Sat Aug 15 20:59:01 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 15 Aug 2015 20:59:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: only manipulate udir,
	sys.path if __main__
Message-ID: <20150815185901.E16E91C0695@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r78992:f83a38b1ce0d
Date: 2015-08-15 21:54 +0300
http://bitbucket.org/pypy/pypy/changeset/f83a38b1ce0d/

Log:	only manipulate udir, sys.path if __main__

diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py
--- a/rpython/jit/backend/tool/viewcode.py
+++ b/rpython/jit/backend/tool/viewcode.py
@@ -17,18 +17,6 @@
 import subprocess
 from bisect import bisect_left
 
-# don't use rpython.tool.udir here to avoid removing old usessions which
-# might still contain interesting executables
-udir = py.path.local.make_numbered_dir(prefix='viewcode-', keep=2)
-tmpfile = str(udir.join('dump.tmp'))
-
-# hack hack
-import rpython.tool
-mod = new.module('rpython.tool.udir')
-mod.udir = udir
-sys.modules['rpython.tool.udir'] = mod
-rpython.tool.udir = mod
-
 # ____________________________________________________________
 # Some support code from Psyco.  There is more over there,
 # I am porting it in a lazy fashion...  See py-utils/xam.py
@@ -438,6 +426,18 @@
 # ____________________________________________________________
 
 if __name__ == '__main__':
+    # don't use rpython.tool.udir here to avoid removing old usessions which
+    # might still contain interesting executables
+    udir = py.path.local.make_numbered_dir(prefix='viewcode-', keep=2)
+    tmpfile = str(udir.join('dump.tmp'))
+
+    # hack hack
+    import rpython.tool
+    mod = new.module('rpython.tool.udir')
+    mod.udir = udir
+    sys.modules['rpython.tool.udir'] = mod
+    rpython.tool.udir = mod
+
     if '--text' in sys.argv:
         sys.argv.remove('--text')
         showgraph = False
@@ -463,3 +463,7 @@
         world.show(showtext=True)
     else:
         world.showtextonly()
+else:
+    from rpython.tool.udir import udir
+    tmpfile = str(udir.join('dump.tmp'))
+    

From noreply at buildbot.pypy.org  Sun Aug 16 09:15:50 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sun, 16 Aug 2015 09:15:50 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: merge default
Message-ID: <20150816071550.1EE2C1C1277@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78993:f22171f122f5
Date: 2015-08-15 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/f22171f122f5/

Log:	merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -66,3 +66,4 @@
 
 .. branch: vmprof-review
 
+Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,4 +1,5 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib.rvmprof import VMProfPlatformUnsupported
 
 class Module(MixedModule):
     """
@@ -19,4 +20,7 @@
 # already found by the annotator to be the original empty
 # method, and the annotator doesn't notice that interp_vmprof.py
 # (loaded later) replaces this method.
-import pypy.module._vmprof.interp_vmprof
+try:
+    import pypy.module._vmprof.interp_vmprof
+except VMProfPlatformUnsupported, e:
+    pass
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -52,11 +52,16 @@
 
     @jit.unroll_safe
     def setslice(self, space, arr):
-        if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
-            raise oefmt(space.w_ValueError,
-                "could not broadcast input array from shape "
-                "(%s) into shape ()",
-                ','.join([str(x) for x in arr.get_shape()]))
+        if len(arr.get_shape()) >  len(self.get_shape()):
+            # record arrays get one extra dimension
+            if not self.dtype.is_record() or \
+                    len(arr.get_shape()) > len(self.get_shape()) + 1:
+                raise oefmt(space.w_ValueError,
+                    "could not broadcast input array from shape "
+                    "(%s) into shape (%s)",
+                    ','.join([str(x) for x in arr.get_shape()]),
+                    ','.join([str(x) for x in self.get_shape()]),
+                    )
         shape = shape_agreement(space, self.get_shape(), arr)
         impl = arr.implementation
         if impl.storage == self.storage:
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -247,6 +247,20 @@
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
+        # TODO: multiarray/mapping.c calls a subclass's __getitem__ here, which
+        # is a big performance hit but necessary for the matrix class. The original
+        # C code is like:
+        #/*
+        #* WARNING: There is a huge special case here. If this is not a
+        #*          base class array, we have to get the view through its
+        #*          very own index machinery.
+        #*          Many subclasses should probably call __setitem__
+        #*          with a base class ndarray view to avoid this.
+        #*/
+        #else if (!(index_type & (HAS_FANCY | HAS_SCALAR_ARRAY))
+        #        && !PyArray_CheckExact(self)) {
+        #view = (PyArrayObject *)PyObject_GetItem((PyObject *)self, ind);
+
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
             self.setitem_filter(space, w_idx, convert_to_array(space, w_value))
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -2668,6 +2668,15 @@
         a[0, :: -1] = numpy.array([11, 12])
         assert (a == [[12, 11], [5, 10]]).all()
 
+        a = numpy.zeros((3, 2), int)
+        b = numpy.ones((3, 1), int)
+        exc = raises(ValueError, 'a[:, 1] = b')
+        assert str(exc.value) == "could not broadcast " +\
+                "input array from shape (3,1) into shape (3)"
+        a[:, 1] = b[:,0] > 0.5
+        assert (a == [[0, 1], [0, 1], [0, 1]]).all()
+        
+
     def test_ufunc(self):
         from numpy import array
         a = array([[1, 2], [3, 4], [5, 6]])
diff --git a/pypy/module/micronumpy/test/test_ufuncs.py b/pypy/module/micronumpy/test/test_ufuncs.py
--- a/pypy/module/micronumpy/test/test_ufuncs.py
+++ b/pypy/module/micronumpy/test/test_ufuncs.py
@@ -1166,6 +1166,7 @@
         assert (logical_xor([True, False, True, False], [1, 2, 0, 0])
                 == [False, True, True, False]).all()
         assert (logical_not([True, False]) == [False, True]).all()
+        assert logical_and.reduce([1.,1.]) == True
 
     def test_logn(self):
         import math
diff --git a/pypy/module/micronumpy/ufuncs.py b/pypy/module/micronumpy/ufuncs.py
--- a/pypy/module/micronumpy/ufuncs.py
+++ b/pypy/module/micronumpy/ufuncs.py
@@ -511,15 +511,15 @@
         W_Ufunc.__init__(self, name, promote_to_largest, promote_to_float, promote_bools,
                          identity, int_only, allow_bool, allow_complex, complex_to_float)
         self.func = func
-        self.bool_result = bool_result
         if name == 'logical_and':
             self.done_func = done_if_false
         elif name == 'logical_or':
             self.done_func = done_if_true
         else:
             self.done_func = None
+        self.bool_result = bool_result or (self.done_func is not None)
         self.simple_binary = (
-            allow_complex and allow_bool and not bool_result and not int_only
+            allow_complex and allow_bool and not self.bool_result and not int_only
             and not complex_to_float and not promote_to_float
             and not promote_bools)
 
@@ -630,7 +630,7 @@
                                             r_dtype.is_complex())):
             raise oefmt(space.w_TypeError,
                 "ufunc '%s' not supported for the input types", self.name)
-        if self.bool_result:
+        if self.bool_result and not self.done_func:
             # XXX: should actually pass the arrays
             dtype = find_result_type(space, [], [l_dtype, r_dtype])
             bool_dtype = get_dtype_cache(space).w_booldtype
diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -28,9 +28,9 @@
             f16 = raw_load(i9, i5, descr=)
             guard_true(i15, descr=...)
             guard_not_invalidated(descr=...)
-            i17 = cast_float_to_int(f16)
-            i19 = int_is_true(i17)
-            guard_true(i19, descr=...)
+            i18 = float_ne(f16, 0.000000)
+            guard_true(i18, descr=...)
+            guard_nonnull_class(p2, ConstClass(W_BoolBox), descr=...)
             i20 = getfield_gc_pure(p2, descr=)
             i21 = int_is_true(i20)
             guard_false(i21, descr=...)
@@ -70,10 +70,10 @@
         assert loop.match("""
             f31 = raw_load(i9, i29, descr=)
             guard_not_invalidated(descr=...)
+            i32 = float_ne(f31, 0.000000)
+            guard_true(i32, descr=...)
             i34 = getarrayitem_raw(#, #, descr=)  # XXX what are these?
             guard_value(i34, #, descr=...)                  # XXX don't appear in
-            i32 = float_ne(f31, 0.000000)
-            guard_true(i32, descr=...)
             i35 = getarrayitem_raw(#, #, descr=)  # XXX equiv test_zjit
             i36 = int_add(i24, 1)
             i37 = int_add(i29, i28)
diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py
--- a/pypy/objspace/std/mapdict.py
+++ b/pypy/objspace/std/mapdict.py
@@ -569,17 +569,19 @@
     rangen = unroll.unrolling_iterable(range(n))
     nmin1 = n - 1
     rangenmin1 = unroll.unrolling_iterable(range(nmin1))
+    valnmin1 = "_value%s" % nmin1
     class subcls(BaseMapdictObject, supercls):
         def _init_empty(self, map):
-            for i in rangen:
-                setattr(self, "_value%s" % i, erase_item(None))
+            for i in rangenmin1:
+                setattr(self, "_value%s" % i, None)
+            setattr(self, valnmin1, erase_item(None))
             self.map = map
 
         def _has_storage_list(self):
             return self.map.length() > n
 
         def _mapdict_get_storage_list(self):
-            erased = getattr(self, "_value%s" % nmin1)
+            erased = getattr(self, valnmin1)
             return unerase_list(erased)
 
         def _mapdict_read_storage(self, storageindex):
@@ -587,23 +589,21 @@
             if storageindex < nmin1:
                 for i in rangenmin1:
                     if storageindex == i:
-                        erased = getattr(self, "_value%s" % i)
-                        return unerase_item(erased)
+                        return getattr(self, "_value%s" % i)
             if self._has_storage_list():
                 return self._mapdict_get_storage_list()[storageindex - nmin1]
             erased = getattr(self, "_value%s" % nmin1)
             return unerase_item(erased)
 
         def _mapdict_write_storage(self, storageindex, value):
-            erased = erase_item(value)
             for i in rangenmin1:
                 if storageindex == i:
-                    setattr(self, "_value%s" % i, erased)
+                    setattr(self, "_value%s" % i, value)
                     return
             if self._has_storage_list():
                 self._mapdict_get_storage_list()[storageindex - nmin1] = value
                 return
-            setattr(self, "_value%s" % nmin1, erased)
+            setattr(self, "_value%s" % nmin1, erase_item(value))
 
         def _mapdict_storage_length(self):
             if self._has_storage_list():
@@ -615,9 +615,9 @@
             len_storage = len(storage)
             for i in rangenmin1:
                 if i < len_storage:
-                    erased = erase_item(storage[i])
+                    erased = storage[i]
                 else:
-                    erased = erase_item(None)
+                    erased = None
                 setattr(self, "_value%s" % i, erased)
             has_storage_list = self._has_storage_list()
             if len_storage < n:
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -195,7 +195,8 @@
             splitted = split(value, self._chr('\t'))
 
         try:
-            ovfcheck(len(splitted) * tabsize)
+            if tabsize > 0:
+                ovfcheck(len(splitted) * tabsize)
         except OverflowError:
             raise oefmt(space.w_OverflowError, "new string is too long")
         expanded = oldtoken = splitted.pop(0)
@@ -210,6 +211,8 @@
     def _tabindent(self, token, tabsize):
         """calculates distance behind the token to the next tabstop"""
 
+        if tabsize <= 0:
+            return 0
         distance = tabsize
         if token:
             distance = 0
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -388,6 +388,10 @@
             skip("Wrong platform")
         raises((MemoryError, OverflowError), 't\tt\t'.expandtabs, sys.maxint)
 
+    def test_expandtabs_0(self):
+        assert 'x\ty'.expandtabs(0) == 'xy'
+        assert 'x\ty'.expandtabs(-42) == 'xy'
+
     def test_splitlines(self):
         s = ""
         assert s.splitlines() == []
diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py
--- a/pypy/objspace/std/test/test_mapdict.py
+++ b/pypy/objspace/std/test/test_mapdict.py
@@ -500,12 +500,12 @@
         obj = objectcls()
         obj.user_setup(space, cls)
         obj.setdictvalue(space, "a", w1)
-        assert unerase_item(obj._value0) is w1
+        assert obj._value0 is w1
         assert obj.getdictvalue(space, "a") is w1
         assert obj.getdictvalue(space, "b") is None
         assert obj.getdictvalue(space, "c") is None
         obj.setdictvalue(space, "a", w2)
-        assert unerase_item(obj._value0) is w2
+        assert obj._value0 is w2
         assert obj.getdictvalue(space, "a") == w2
         assert obj.getdictvalue(space, "b") is None
         assert obj.getdictvalue(space, "c") is None
@@ -523,7 +523,7 @@
 
         res = obj.deldictvalue(space, "a")
         assert res
-        assert unerase_item(obj._value0) is w4
+        assert obj._value0 is w4
         assert obj.getdictvalue(space, "a") is None
         assert obj.getdictvalue(space, "b") is w4
         assert obj.getdictvalue(space, "c") is None
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -494,6 +494,10 @@
             skip("Wrong platform")
         raises((OverflowError, MemoryError), u't\tt\t'.expandtabs, sys.maxint)
 
+    def test_expandtabs_0(self):
+        assert u'x\ty'.expandtabs(0) == u'xy'
+        assert u'x\ty'.expandtabs(-42) == u'xy'
+
     def test_translate(self):
         assert u'bbbc' == u'abababc'.translate({ord('a'):None})
         assert u'iiic' == u'abababc'.translate({ord('a'):None, ord('b'):ord('i')})
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -79,9 +79,6 @@
             assert methname == 'jit_merge_point', (
                 "reds='auto' is supported only for jit drivers which "
                 "calls only jit_merge_point. Found a call to %s" % methname)
-            if jitdriver.numreds is not None:
-                raise AssertionError("there are multiple jit_merge_points "
-                                     "with the same jitdriver")
             #
             # compute the set of live variables across the jit_marker
             alive_v = set()
@@ -99,7 +96,11 @@
                                            v.concretetype is not lltype.Void]
             reds_v = sort_vars(reds_v)
             op.args.extend(reds_v)
-            jitdriver.numreds = len(reds_v)
+            if jitdriver.numreds is None:
+                jitdriver.numreds = len(reds_v)
+            elif jitdriver.numreds != len(reds_v):
+                raise AssertionError("there are multiple jit_merge_points "
+                                     "with the same jitdriver")
 
 def split_before_jit_merge_point(graph, portalblock, portalopindex):
     """Split the block just before the 'jit_merge_point',
diff --git a/rpython/jit/metainterp/heapcache.py b/rpython/jit/metainterp/heapcache.py
--- a/rpython/jit/metainterp/heapcache.py
+++ b/rpython/jit/metainterp/heapcache.py
@@ -9,6 +9,7 @@
 
     def reset_keep_likely_virtual(self):
         self.known_class = False
+        self.known_nullity = False
         # did we see the allocation during tracing?
         self.seen_allocation = False
         self.is_unescaped = False
@@ -290,6 +291,15 @@
     def class_now_known(self, box):
         self.getvalue(box).known_class = True
 
+    def is_nullity_known(self, box):
+        value = self.getvalue(box, create=False)
+        if value:
+            return value.known_nullity
+        return False
+
+    def nullity_now_known(self, box):
+        self.getvalue(box).known_nullity = True
+
     def is_nonstandard_virtualizable(self, box):
         value = self.getvalue(box, create=False)
         if value:
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -369,7 +369,10 @@
         ).compile()
 
     def _establish_nullity(self, box, orgpc):
+        heapcache = self.metainterp.heapcache
         value = box.nonnull()
+        if heapcache.is_nullity_known(box):
+            return value
         if value:
             if not self.metainterp.heapcache.is_class_known(box):
                 self.metainterp.generate_guard(rop.GUARD_NONNULL, box,
@@ -380,6 +383,7 @@
                                                resumepc=orgpc)
                 promoted_box = box.constbox()
                 self.metainterp.replace_box(box, promoted_box)
+        heapcache.nullity_now_known(box)
         return value
 
     @arguments("box", "label", "orgpc")
diff --git a/rpython/jit/metainterp/test/test_heapcache.py b/rpython/jit/metainterp/test/test_heapcache.py
--- a/rpython/jit/metainterp/test/test_heapcache.py
+++ b/rpython/jit/metainterp/test/test_heapcache.py
@@ -69,6 +69,19 @@
         assert not h.is_class_known(1)
         assert not h.is_class_known(2)
 
+    def test_known_nullity(self):
+        h = HeapCache()
+        assert not h.is_nullity_known(1)
+        assert not h.is_nullity_known(2)
+        h.nullity_now_known(1)
+        assert h.is_nullity_known(1)
+        assert not h.is_nullity_known(2)
+
+        h.reset()
+        assert not h.is_nullity_known(1)
+        assert not h.is_nullity_known(2)
+
+
     def test_nonstandard_virtualizable(self):
         h = HeapCache()
         assert not h.is_nonstandard_virtualizable(1)
diff --git a/rpython/jit/metainterp/test/test_tracingopts.py b/rpython/jit/metainterp/test/test_tracingopts.py
--- a/rpython/jit/metainterp/test/test_tracingopts.py
+++ b/rpython/jit/metainterp/test/test_tracingopts.py
@@ -107,6 +107,28 @@
         assert res == -7 * 2
         self.check_operations_history(getfield_gc=1)
 
+    def test_heap_caching_nonnull(self):
+        class A:
+            def __init__(self, x=None):
+                self.next = x
+        a0 = A()
+        a1 = A()
+        a2 = A(a1)
+        def fn(n):
+            if n > 0:
+                a = a1
+            else:
+                a = a2
+            if a.next:
+                a = A(a.next)
+                result = a.next is not None
+                a0.next = a
+                return result
+            return False
+        res = self.interp_operations(fn, [-7])
+        assert res == True
+        self.check_operations_history(guard_nonnull=1)
+
     def test_heap_caching_while_tracing_invalidation(self):
         class A:
             pass
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -725,10 +725,15 @@
     result_w = []
     #
     if not we_are_translated():   # fast path before translation
-        for gcref in roots:       # 'roots' is all objects in this case
-            w_obj = callback(gcref)
-            if w_obj is not None:
-                result_w.append(w_obj)
+        seen = set()
+        while roots:
+            gcref = roots.pop()
+            if gcref not in seen:
+                seen.add(gcref)
+                w_obj = callback(gcref)
+                if w_obj is not None:
+                    result_w.append(w_obj)
+                roots.extend(get_rpy_referents(gcref))
         return result_w
     #
     pending = roots[:]
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -2,6 +2,7 @@
 from rpython.rlib.rvmprof.rvmprof import _get_vmprof, VMProfError
 from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code, MAX_FUNC_NAME
 from rpython.rlib.rvmprof.rvmprof import _was_registered
+from rpython.rlib.rvmprof.cintf import VMProfPlatformUnsupported
 
 #
 # See README.txt.
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -1,6 +1,7 @@
 import py
 import sys
 from rpython.tool.udir import udir
+from rpython.tool.version import rpythonroot
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.rtyper.tool import rffi_platform as platform
@@ -16,7 +17,7 @@
                                         " x86-64 CPUs for now")
 
 
-    ROOT = py.path.local(__file__).join('..')
+    ROOT = py.path.local(rpythonroot).join('rpython', 'rlib', 'rvmprof')
     SRC = ROOT.join('src')
 
 
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -196,6 +196,11 @@
     (including 'self' if applicable).
     """
     def decorate(func):
+        try:
+            _get_vmprof()
+        except cintf.VMProfPlatformUnsupported:
+            return func
+
         if hasattr(func, 'im_self'):
             assert func.im_self is None
             func = func.im_func
diff --git a/rpython/rlib/rvmprof/test/test_rvmprof.py b/rpython/rlib/rvmprof/test/test_rvmprof.py
--- a/rpython/rlib/rvmprof/test/test_rvmprof.py
+++ b/rpython/rlib/rvmprof/test/test_rvmprof.py
@@ -8,7 +8,10 @@
 
     class MyCode:
         pass
-    rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    try:
+        rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    except rvmprof.VMProfPlatformUnsupported:
+        pass
 
     @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
     def main(code, num):
@@ -29,7 +32,10 @@
 
     class MyCode:
         pass
-    rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    try:
+        rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    except rvmprof.VMProfPlatformUnsupported:
+        pass
 
     class A:
         pass
@@ -54,7 +60,10 @@
 
     class MyCode:
         pass
-    rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    try:
+        rvmprof.register_code_object_class(MyCode, lambda code: 'some code')
+    except rvmprof.VMProfPlatformUnsupported, e:
+        py.test.skip(str(e))
 
     @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
     def main(code, num):
@@ -79,7 +88,10 @@
         pass
     def get_name(code):
         return 'py:code:52:x'
-    rvmprof.register_code_object_class(MyCode, get_name)
+    try:
+        rvmprof.register_code_object_class(MyCode, get_name)
+    except rvmprof.VMProfPlatformUnsupported, e:
+        py.test.skip(str(e))
 
     @rvmprof.vmprof_execute_code("xcode1", lambda code, num: code)
     def main(code, num):
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,4 @@
-import time, os
+import time, os, sys, py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
 from rpython.translator.c.test.test_genc import compile
@@ -14,7 +14,10 @@
 
 
 def setup_module(mod):
-    rvmprof.register_code_object_class(MyCode, MyCode.get_name)
+    try:
+        rvmprof.register_code_object_class(MyCode, MyCode.get_name)
+    except rvmprof.VMProfPlatformUnsupported, e:
+        py.test.skip(str(e))
 
 
 @rvmprof.vmprof_execute_code("interp", lambda code: code)
@@ -52,7 +55,9 @@
     return main
 
 def test_interpreted():
-    main()
+    # takes forever if the Python process is already big...
+    import subprocess
+    subprocess.check_call([sys.executable, __file__])
 
 def test_compiled():
     fn = compile(main, [], gcpolicy="minimark")
@@ -60,3 +65,8 @@
         os.unlink(PROF_FILE)
     fn()
     assert os.path.exists(PROF_FILE)
+
+if __name__ == '__main__':
+    setup_module(None)
+    res = main()
+    assert res == 0

From noreply at buildbot.pypy.org  Sun Aug 16 09:15:52 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sun, 16 Aug 2015 09:15:52 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: make record_exact_class also
 imply that the box is not null
Message-ID: <20150816071552.3C8551C1277@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78994:4c2707c6443a
Date: 2015-08-15 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/4c2707c6443a/

Log:	make record_exact_class also imply that the box is not null

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -335,6 +335,7 @@
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         if not isinstance(expectedclassbox, Const):
+            value.make_nonnull(None)
             # can't optimize
             return
         realclassbox = value.get_constant_class(self.optimizer.cpu)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7089,6 +7089,7 @@
         [p0]
         p1 = getfield_gc(p0, descr=nextdescr)
         record_exact_class(p1, ConstClass(node_vtable))
+        guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable)) []
         jump(p1)
         """
@@ -7098,10 +7099,12 @@
         jump(p1)
         """
         self.optimize_loop(ops, expected)
+
         ops = """
         [p0, i0]
         p1 = getfield_gc(p0, descr=nextdescr)
         record_exact_class(p1, i0)
+        guard_nonnull(p1) []
         guard_class(p1, ConstClass(node_vtable)) []
         jump(p1, i0)
         """
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -282,6 +282,7 @@
         adr = clsbox.getaddr()
         self.execute(rop.RECORD_EXACT_CLASS, box, clsbox)
         self.metainterp.heapcache.class_now_known(box)
+        self.metainterp.heapcache.nullity_now_known(box)
 
     @arguments("box")
     def _opimpl_any_return(self, box):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1085,14 +1085,13 @@
     Assure the JIT that value is an instance of cls. This is a precise
     class check, like a guard_class.
     """
-    assert type(value) is cls
+    assert value is not None and type(value) is cls
 
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_class
 
     def compute_result_annotation(self, s_inst, s_cls):
         from rpython.annotator import model as annmodel
-        assert not s_inst.can_be_none()
         assert isinstance(s_inst, annmodel.SomeInstance)
 
     def specialize_call(self, hop):

From noreply at buildbot.pypy.org  Sun Aug 16 09:15:54 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Sun, 16 Aug 2015 09:15:54 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: track when lists store
 instances of the same (rpython level) class
Message-ID: <20150816071554.5E6BA1C1277@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r78995:c0aea2847f70
Date: 2015-08-15 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/c0aea2847f70/

Log:	track when lists store instances of the same (rpython level) class

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -40,6 +40,19 @@
 
 UNROLL_CUTOFF = 5
 
+class FixedClsStrategyCache(object):
+    def __init__(self, space):
+        self.space = space
+        self.generic_strategy = space.fromcache(ObjectListStrategy)
+        self.cache = {}
+
+    @jit.elidable
+    def get_or_build(self, cls):
+        strategy = self.cache.get(cls, None)
+        if strategy is None:
+            strategy = ObjectListStrategy(self.space, cls)
+            self.cache[cls] = strategy
+        return strategy
 
 def make_range_list(space, start, step, length):
     if length <= 0:
@@ -125,7 +138,15 @@
         else:
             return space.fromcache(IntOrFloatListStrategy)
 
-    return space.fromcache(ObjectListStrategy)
+    cache = space.fromcache(FixedClsStrategyCache)
+
+    # check for all the same class
+    for i in range(1, len(list_w)):
+        if type(list_w[i]) is not type(w_firstobj):
+            break
+    else:
+        return cache.get_or_build(type(w_firstobj))
+    return cache.generic_strategy
 
 
 def _get_printable_location(w_type):
@@ -685,7 +706,7 @@
             if has_key:
                 sorterclass = CustomKeySort
             else:
-                if self.strategy is space.fromcache(ObjectListStrategy):
+                if isinstance(self.strategy, ObjectListStrategy):
                     sorterclass = SimpleSort
                 else:
                     self.sort(reverse)
@@ -936,7 +957,8 @@
         elif type(w_item) is W_FloatObject:
             strategy = self.space.fromcache(FloatListStrategy)
         else:
-            strategy = self.space.fromcache(ObjectListStrategy)
+            cache = self.space.fromcache(FixedClsStrategyCache)
+            strategy = cache.get_or_build(type(w_item))
 
         storage = strategy.get_empty_storage(self.get_sizehint())
         w_list.strategy = strategy
@@ -1418,9 +1440,8 @@
         w_list.append(w_item)
 
     def insert(self, w_list, index, w_item):
-        l = self.unerase(w_list.lstorage)
-
         if self.is_correct_type(w_item):
+            l = self.unerase(w_list.lstorage)
             l.insert(index, self.unwrap(w_item))
             return
 
@@ -1428,8 +1449,8 @@
         w_list.insert(index, w_item)
 
     def _extend_from_list(self, w_list, w_other):
-        l = self.unerase(w_list.lstorage)
         if self.list_is_correct_type(w_other):
+            l = self.unerase(w_list.lstorage)
             l += self.unerase(w_other.lstorage)
             return
         elif w_other.strategy.is_empty_strategy():
@@ -1581,14 +1602,30 @@
 
 
 class ObjectListStrategy(ListStrategy):
+    _immutable_fields_ = ['_known_cls']
+
     import_from_mixin(AbstractUnwrappedStrategy)
 
     _none_value = None
 
+    def __init__(self, space, known_cls=None):
+        self.space = space
+        self._known_cls = known_cls
+
+    def __repr__(self):
+        return "ObjectListStrategy(space, %s)" % (self._known_cls, )
+
+    @jit.elidable_promote('0')
+    def get_known_cls(self):
+        return self._known_cls
+
     def unwrap(self, w_obj):
         return w_obj
 
     def wrap(self, item):
+        cls = self.get_known_cls()
+        if cls is not None:
+            jit.record_exact_class(item, cls)
         return item
 
     erase, unerase = rerased.new_erasing_pair("object")
@@ -1596,10 +1633,13 @@
     unerase = staticmethod(unerase)
 
     def is_correct_type(self, w_obj):
+        cls = self.get_known_cls()
+        if cls is not None:
+            return type(w_obj) is cls
         return True
 
     def list_is_correct_type(self, w_list):
-        return w_list.strategy is self.space.fromcache(ObjectListStrategy)
+        return w_list.strategy is self
 
     def init_from_list_w(self, w_list, list_w):
         w_list.lstorage = self.erase(list_w)
diff --git a/pypy/objspace/std/test/test_liststrategies.py b/pypy/objspace/std/test/test_liststrategies.py
--- a/pypy/objspace/std/test/test_liststrategies.py
+++ b/pypy/objspace/std/test/test_liststrategies.py
@@ -8,6 +8,8 @@
 from pypy.objspace.std import listobject
 from pypy.objspace.std.test.test_listobject import TestW_ListObject
 
+from rpython.rlib import jit
+
 
 class TestW_ListStrategies(TestW_ListObject):
     def test_check_strategy(self):
@@ -1035,6 +1037,93 @@
             (int, 5), (float, 1.2), (int, 1), (float, 1.0)]
 
 
+class TestKnownClasListStrategy:
+    def test_create_from_list(self):
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup])
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert w_l.strategy._known_cls is type(w_tup)
+
+        w_l1 = W_ListObject(space, [w_tup, w_tup])
+        assert w_l1.strategy is w_l.strategy
+
+    def test_create_empty(self):
+        space = self.space
+        w_l = W_ListObject(space, [])
+        w_tup = space.wrap((1, 2))
+        w_l.append(w_tup)
+        assert isinstance(w_l.strategy, ObjectListStrategy)
+        assert w_l.strategy._known_cls is type(w_tup)
+        w_l.append(w_tup)
+        assert w_l.strategy._known_cls is type(w_tup)
+
+    def test_append(self):
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup])
+        strategy = w_l.strategy
+        w_l.append(w_tup)
+        assert strategy is w_l.strategy
+        w_l.append(w_l)
+        assert w_l.strategy.get_known_cls() is None
+
+    def test_setitem(self):
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup])
+        w_l.setitem(0, w_tup)
+        assert w_l.strategy.get_known_cls() is type(w_tup)
+        w_l.setitem(0, w_l)
+        assert w_l.strategy.get_known_cls() is None
+
+    def test_extend(self):
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup])
+        w_l1 = W_ListObject(space, [w_tup, w_tup])
+        w_l.extend(w_l1)
+        assert w_l.strategy.get_known_cls() is type(w_tup)
+
+        w_l2 = W_ListObject(space, [w_l, w_l])
+        w_l.extend(w_l2)
+        assert w_l.strategy.get_known_cls() is None
+
+    def test_getslice(self):
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup, w_tup])
+        w_l1 = w_l.getslice(0, 1, 1, 1)
+        assert w_l.strategy is w_l1.strategy
+
+    def test_setslice(self):
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup, w_tup])
+        w_l.setslice(0, 1, 2, W_ListObject(space, [w_tup, w_tup, w_tup]))
+        assert w_l.strategy.get_known_cls() is type(w_tup)
+        w_l.setslice(0, 1, 2, W_ListObject(space, [w_l, w_tup, w_tup]))
+        assert w_l.strategy.get_known_cls() is None
+
+    def test_propagate_cls_knowledge(self, monkeypatch):
+        l = []
+        def record_exact_class(obj, cls):
+            assert type(obj) is cls
+            l.append((obj, cls))
+        monkeypatch.setattr(jit, 'record_exact_class', record_exact_class)
+
+        space = self.space
+        w_tup = space.wrap((1, 2))
+        w_l = W_ListObject(space, [w_tup, w_tup, w_tup])
+        w_tup1 = w_l.getitem(0)
+        assert w_tup1 is w_tup
+        assert l == [(w_tup, type(w_tup))] * 1
+
+        w_tup1 = w_l.pop_end()
+        assert w_tup1 is w_tup
+        assert l == [(w_tup, type(w_tup))] * 2
+
+
 class TestW_ListStrategiesDisabled:
     spaceconfig = {"objspace.std.withliststrategies": False}
 

From noreply at buildbot.pypy.org  Sun Aug 16 16:03:41 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 16 Aug 2015 16:03:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Implement time.monotonic(). Windows and
 Darwin support untested.
Message-ID: <20150816140341.936021C1277@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r78996:10ea3f73852a
Date: 2015-08-16 16:03 +0200
http://bitbucket.org/pypy/pypy/changeset/10ea3f73852a/

Log:	Implement time.monotonic(). Windows and Darwin support untested.

diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -19,6 +19,7 @@
         'strftime': 'interp_time.strftime',
         'sleep' : 'interp_time.sleep',
         '_STRUCT_TM_ITEMS': 'space.wrap(interp_time._STRUCT_TM_ITEMS)',
+        'monotonic': 'interp_time.monotonic',
     }
 
     if os.name == "posix":
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -145,6 +145,12 @@
         ("tm_mon", rffi.INT), ("tm_year", rffi.INT), ("tm_wday", rffi.INT),
         ("tm_yday", rffi.INT), ("tm_isdst", rffi.INT)])
 
+if sys.platform == 'darwin':
+    CConfig.TIMEBASE_INFO = platform.Struct("struct mach_timebase_info", [
+        ("numer", rffi.UINT),
+        ("denom", rffi.UINT),
+    ])
+
 # XXX: optionally support the 2 additional tz fields
 _STRUCT_TM_ITEMS = 9
 
@@ -710,3 +716,38 @@
         finally:
             lltype.free(outbuf, flavor='raw')
         i += i
+
+
+if _WIN:
+    # untested so far
+    _GetTickCount64 = rwin32.winexternal('GetTickCount64', [], rffi.ULONGLONG)
+
+    def monotonic(space):
+        return space.wrap(_GetTickCount64() * 1e-3)
+
+elif sys.platform == 'darwin':
+    # untested so far
+    c_mach_timebase_info = external('mach_timebase_info',
+                                    [lltype.Ptr(TIMEBASE_INFO)], lltype.Void)
+    c_mach_absolute_time = external('mach_absolute_time', [], lltype.ULONGLONG)
+
+    timebase_info = lltype.malloc(TIMEBASE_INFO, flavor='raw', zero=True,
+                                  immortal=True)
+
+    def monotonic():
+        if timebase_info.denom == 0:
+            mach_timebase_info(timebase_info)
+        time = mach_absolute_time()
+        nanosecs = time * timebase_info.numer / timebase_info.denom
+        secs = nanosecs / 10**9
+        rest = nanosecs % 10**9
+        return space.wrap(float(secs) + float(rest) * 1e-9)
+
+else:
+    assert _POSIX
+    if cConfig.CLOCK_HIGHRES is not None:
+        def monotonic(space):
+            return clock_gettime(space, cConfig.CLOCK_HIGHRES)
+    else:
+        def monotonic(space):
+            return clock_gettime(space, cConfig.CLOCK_MONOTONIC)
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -358,3 +358,11 @@
         new = pickle.loads(pickle.dumps(now))
         assert new == now
         assert type(new) is type(now)
+
+    def test_monotonic(self):
+        import time
+        t1 = time.monotonic()
+        assert isinstance(t1, float)
+        time.sleep(0.02)
+        t2 = time.monotonic()
+        assert t1 < t2

From noreply at buildbot.pypy.org  Sun Aug 16 20:10:04 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sun, 16 Aug 2015 20:10:04 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Implement time.perf_counter().
Message-ID: <20150816181004.A65C11C06AD@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r78997:4d2cde9414b9
Date: 2015-08-16 20:07 +0200
http://bitbucket.org/pypy/pypy/changeset/4d2cde9414b9/

Log:	Implement time.perf_counter().

diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -20,6 +20,7 @@
         'sleep' : 'interp_time.sleep',
         '_STRUCT_TM_ITEMS': 'space.wrap(interp_time._STRUCT_TM_ITEMS)',
         'monotonic': 'interp_time.monotonic',
+        'perf_counter': 'interp_time.perf_counter',
     }
 
     if os.name == "posix":
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.rtime import c_clock_gettime, TIMESPEC
+from rpython.rlib.rtime import c_clock_gettime, TIMESPEC, win_perf_counter
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 import math
@@ -751,3 +751,12 @@
     else:
         def monotonic(space):
             return clock_gettime(space, cConfig.CLOCK_MONOTONIC)
+
+
+if _WIN:
+    def perf_counter(space):
+        return space.wrap(win_perf_counter())
+
+else:
+    def perf_counter(space):
+        return monotonic(space)
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -366,3 +366,7 @@
         time.sleep(0.02)
         t2 = time.monotonic()
         assert t1 < t2
+
+    def test_perf_counter(self):
+        import time
+        assert isinstance(time.perf_counter(), float)
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -175,19 +175,22 @@
                            lltype.Void,
                            releasegil=False)
 
+def win_perf_counter():
+    a = lltype.malloc(A, flavor='raw')
+    if state.divisor == 0.0:
+        QueryPerformanceCounter(a)
+        state.counter_start = a[0]
+        QueryPerformanceFrequency(a)
+        state.divisor = float(a[0])
+    QueryPerformanceCounter(a)
+    diff = a[0] - state.counter_start
+    lltype.free(a, flavor='raw')
+    return float(diff) / state.divisor
+
 @replace_time_function('clock')
 def clock():
     if _WIN32:
-        a = lltype.malloc(A, flavor='raw')
-        if state.divisor == 0.0:
-            QueryPerformanceCounter(a)
-            state.counter_start = a[0]
-            QueryPerformanceFrequency(a)
-            state.divisor = float(a[0])
-        QueryPerformanceCounter(a)
-        diff = a[0] - state.counter_start
-        lltype.free(a, flavor='raw')
-        return float(diff) / state.divisor
+        return win_perf_counter()
     elif CLOCK_PROCESS_CPUTIME_ID is not None:
         with lltype.scoped_alloc(TIMESPEC) as a:
             c_clock_gettime(CLOCK_PROCESS_CPUTIME_ID, a)

From noreply at buildbot.pypy.org  Sun Aug 16 20:10:42 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 16 Aug 2015 20:10:42 +0200 (CEST)
Subject: [pypy-commit] buildbot default: add cffi import library copy step
	to TranslatedTest
Message-ID: <20150816181042.2825B1C06AD@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r955:3275430ea6da
Date: 2015-08-16 21:11 +0300
http://bitbucket.org/pypy/buildbot/changeset/3275430ea6da/

Log:	add cffi import library copy step to TranslatedTest

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -666,10 +666,15 @@
             haltOnFailure=True,
             workdir='.'))
         self.addStep(ShellCmd(
-            description="move ctypes resource cache",
+            description="copy ctypes resource cache",
             command=['cp', '-rv', 'pypy-c/lib_pypy/ctypes_config_cache', 'build/lib_pypy'],
             haltOnFailure=True,
             workdir='.'))
+        self.addStep(ShellCmd(
+            description="copy cffi import libraries"
+            command=['cp', '-rv', 'pypy-c/lib_pypy/*.so', 'build/lib_pypy'],
+            haltOnFailure=True,
+            workdir='.'))
 
         add_translated_tests(self, prefix, platform, app_tests, lib_python, pypyjit)
 
diff --git a/bot2/pypybuildbot/master.py b/bot2/pypybuildbot/master.py
--- a/bot2/pypybuildbot/master.py
+++ b/bot2/pypybuildbot/master.py
@@ -279,7 +279,7 @@
         ], branch=None, hour=1, minute=0),
 
         Triggerable("NUMPY64_scheduler", [
-            NUMPY_64,                  # on tannit64, uses 1 core, takes about 15min.
+            NUMPY_64,                  # on tannit64, uses 1 core, takes about 5min.
         ]),
 
         Triggerable("NUMPYWIN_scheduler", [

From noreply at buildbot.pypy.org  Sun Aug 16 20:31:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 16 Aug 2015 20:31:26 +0200 (CEST)
Subject: [pypy-commit] pypy default: if loading of .pyc file fails,
 just give up instead of exploding
Message-ID: <20150816183126.B11A11C123B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r78998:b92ab1013bf2
Date: 2015-08-16 20:31 +0200
http://bitbucket.org/pypy/pypy/changeset/b92ab1013bf2/

Log:	if loading of .pyc file fails, just give up instead of exploding

diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("

Author: Ronan Lamy 
Branch: callfamily
Changeset: r78999:3a61e4b83097
Date: 2015-08-16 21:00 +0100
http://bitbucket.org/pypy/pypy/changeset/3a61e4b83097/

Log:	hg merge default

diff too long, truncating to 2000 out of 70765 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -10,3 +10,8 @@
 32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1
 10f1b29a2bd21f837090286174a9ca030b8680b2 release-2.5.0
 9c4588d731b7fe0b08669bd732c2b676cb0a8233 release-2.5.1
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+fcdb941565156385cbac04cfb891f8f4c7a92ef6 release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
+295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -38,8 +38,8 @@
   Armin Rigo
   Maciej Fijalkowski
   Carl Friedrich Bolz
+  Amaury Forgeot d'Arc
   Antonio Cuni
-  Amaury Forgeot d'Arc
   Samuele Pedroni
   Alex Gaynor
   Brian Kearns
@@ -50,9 +50,9 @@
   Holger Krekel
   Christian Tismer
   Hakan Ardo
-  Benjamin Peterson
   Manuel Jacob
   Ronan Lamy
+  Benjamin Peterson
   Anders Chrigstrom
   Eric van Riet Paap
   Wim Lavrijsen
@@ -63,8 +63,8 @@
   Sven Hager
   Anders Lehmann
   Aurelien Campeas
+  Remi Meier
   Niklaus Haldimann
-  Remi Meier
   Camillo Bruni
   Laura Creighton
   Toon Verwaest
@@ -76,10 +76,10 @@
   David Edelsohn
   Anders Hammarquist
   Jakub Gustak
+  Gregor Wegberg
   Guido Wesdorp
   Lawrence Oluyede
   Bartosz Skowron
-  Gregor Wegberg
   Daniel Roberts
   Niko Matsakis
   Adrien Di Mascio
@@ -87,10 +87,11 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
+  Richard Plangger
   Alex Martelli
   Michal Bendowski
+  stian
   Jan de Mooij
-  stian
   Tyler Wade
   Michael Foord
   Stephan Diehl
@@ -133,15 +134,15 @@
   Georg Brandl
   Bert Freudenberg
   Stian Andreassen
+  Edd Barrett
   Wanja Saatkamp
   Gerald Klix
   Mike Blume
+  Tobias Pape
   Oscar Nierstrasz
   Stefan H. Muller
-  Edd Barrett
   Jeremy Thurgood
   Rami Chowdhury
-  Tobias Pape
   Eugene Oden
   Henry Mason
   Vasily Kuznetsov
@@ -167,11 +168,13 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
+  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
   Karl Bartel
   Wouter van Heyst
+  Sebastian Pawluś
   Brian Dorsey
   Victor Stinner
   Andrews Medina
@@ -188,6 +191,7 @@
   Neil Shepperd
   Stanislaw Halik
   Mikael Schönenberg
+  Berkin Ilbeyi
   Elmo M?ntynen
   Jonathan David Riehl
   Anders Qvist
@@ -211,11 +215,11 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
-  Sebastian Pawluś
   Gabriel
   Lukas Vacek
   Andrew Dalke
   Sylvain Thenault
+  Jakub Stasiak
   Nathan Taylor
   Vladimir Kryachko
   Jacek Generowicz
@@ -242,6 +246,7 @@
   Tomo Cocoa
   Toni Mattis
   Lucas Stadler
+  Julian Berman
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -253,6 +258,8 @@
   Ben Darnell
   Roberto De Ioris
   Juan Francisco Cantero Hurtado
+  Ruochen Huang
+  Jeong YunWon
   Godefroid Chappelle
   Joshua Gilbert
   Dan Colish
@@ -271,6 +278,7 @@
   Christian Muirhead
   Berker Peksag
   James Lan
+  Volodymyr Vladymyrov
   shoma hosaka
   Daniel Neuhäuser
   Ben Mather
@@ -316,6 +324,7 @@
   yasirs
   Michael Chermside
   Anna Ravencroft
+  Andrey Churin
   Dan Crosta
   Julien Phalip
   Roman Podoliaka
@@ -420,3 +429,10 @@
 the terms of the GPL license version 2 or any later version.  Thus the
 gdbm module, provided in the file lib_pypy/gdbm.py, is redistributed
 under the terms of the GPL license as well.
+
+License for 'pypy/module/_vmprof/src'
+--------------------------------------
+
+The code is based on gperftools. You may see a copy of the License for it at
+
+    https://code.google.com/p/gperftools/source/browse/COPYING
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("<@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')

From noreply at buildbot.pypy.org  Mon Aug 17 09:55:04 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 09:55:04 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-iterator-sharing: activated
 get/setarrayitem_gc if their descriptor uses primitive data (int/float)
Message-ID: <20150817075504.73DD81C136F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-iterator-sharing
Changeset: r79000:40eb3eef0bac
Date: 2015-08-17 09:55 +0200
http://bitbucket.org/pypy/pypy/changeset/40eb3eef0bac/

Log:	activated get/setarrayitem_gc if their descriptor uses primitive
	data (int/float) added test to ensure this is working properly,
	modified some tests that assumed the old behaviour

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -38,6 +38,10 @@
     states = [out_state,left_state,right_state]
     left_index = 1
     right_index = 2
+    # 1) get rid of scalar cases
+    # 2) switch through cases
+    # 3) you dont need states
+    # 4) no left_right_index
     # left == right == out
     # left == right
     # left == out
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -155,6 +155,13 @@
     def __repr__(self):
         return 'ArrayDescr(%r)' % (self.OUTERA,)
 
+    def is_array_of_primitives(self):
+        kind = getkind(self.A.OF)
+        return kind == 'float' or \
+               kind == 'int' or \
+               kind == ''
+
+
     def is_array_of_pointers(self):
         return getkind(self.A.OF) == 'ref'
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -203,6 +203,11 @@
     def getconcrete_type(self):
         return self.concrete_type
 
+    def is_array_of_primitives(self):
+        return self.flag == FLAG_FLOAT or \
+               self.flag == FLAG_SIGNED or \
+               self.flag == FLAG_UNSIGNED
+
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
 
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -344,6 +344,8 @@
                          rop.VEC_RAW_STORE,
                          rop.VEC_GETARRAYITEM_RAW,
                          rop.VEC_SETARRAYITEM_RAW,
+                         rop.VEC_GETARRAYITEM_GC,
+                         rop.VEC_SETARRAYITEM_GC,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
             if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST:
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -805,8 +805,9 @@
     def operation_{name}(self, op, node):
         descr = op.getdescr()
         idx_ref = self.get_or_create(op.getarg(1))
-        node.memory_ref = MemoryRef(op, idx_ref, {raw_access})
-        self.memory_refs[node] = node.memory_ref
+        if descr.is_array_of_primitives():
+            node.memory_ref = MemoryRef(op, idx_ref, {raw_access})
+            self.memory_refs[node] = node.memory_ref
     """
     exec py.code.Source(array_access_source
            .format(name='RAW_LOAD',raw_access=True)).compile()
@@ -816,6 +817,10 @@
            .format(name='GETARRAYITEM_RAW',raw_access=False)).compile()
     exec py.code.Source(array_access_source
            .format(name='SETARRAYITEM_RAW',raw_access=False)).compile()
+    exec py.code.Source(array_access_source
+           .format(name='GETARRAYITEM_GC',raw_access=False)).compile()
+    exec py.code.Source(array_access_source
+           .format(name='SETARRAYITEM_GC',raw_access=False)).compile()
     del array_access_source
 integral_dispatch_opt = make_dispatcher_method(IntegralForwardModification, 'operation_')
 IntegralForwardModification.inspect_operation = integral_dispatch_opt
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -698,8 +698,10 @@
 
     rop.VEC_RAW_LOAD:         LOAD_TRANS,
     rop.VEC_GETARRAYITEM_RAW: LOAD_TRANS,
+    rop.VEC_GETARRAYITEM_GC: LOAD_TRANS,
     rop.VEC_RAW_STORE:        STORE_TRANS,
     rop.VEC_SETARRAYITEM_RAW: STORE_TRANS,
+    rop.VEC_SETARRAYITEM_GC: STORE_TRANS,
 
     rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE_2, PT_FLOAT_2),
     rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -212,21 +212,6 @@
         """
         self.assert_vectorize(self.parse_loop(ops), self.parse_loop(ops))
 
-    def test_vectorize_skip_impossible_2(self):
-        ops = """
-        [p0,i0]
-        i1 = int_add(i0,1)
-        i2 = int_le(i1, 10)
-        guard_true(i2) []
-        i3 = getarrayitem_gc(p0,i0,descr=intarraydescr)
-        jump(p0,i1)
-        """
-        try:
-            self.vectorize(self.parse_loop(ops))
-            py.test.fail("should not happend")
-        except NotAVectorizeableLoop:
-            pass
-
     def test_unroll_empty_stays_empty(self):
         """ has no operations in this trace, thus it stays empty
         after unrolling it 2 times """
@@ -264,6 +249,26 @@
         """
         self.assert_vectorize(self.parse_loop(ops), self.parse_loop(ops))
 
+    def test_load_primitive_python_list(self):
+        """ it currently rejects pointer arrays """
+        ops = """
+        [p0,i0]
+        i2 = getarrayitem_gc(p0,i0,descr=floatarraydescr)
+        i1 = int_add(i0,1)
+        i3 = getarrayitem_gc(p0,i1,descr=floatarraydescr)
+        i4 = int_add(i1,1)
+        jump(p0,i4)
+        """
+        opt = """
+        [p0,i0]
+        i1 = int_add(i0,1)
+        i2 = int_add(i0,2)
+        i3 = vec_getarrayitem_gc(p0,i0,2,descr=floatarraydescr)
+        jump(p0,i2)
+        """
+        vopt = self.vectorize(self.parse_loop(ops),0)
+        self.assert_equal(vopt.loop, self.parse_loop(opt))
+
     def test_vect_unroll_char(self):
         """ a 16 byte vector register can hold 16 bytes thus 
         it is unrolled 16 times. (it is the smallest type in the trace) """
@@ -316,7 +321,7 @@
     def test_estimate_unroll_factor_smallest_byte_zero(self):
         ops = """
         [p0,i0]
-        raw_load(p0,i0,descr=arraydescr2)
+        raw_load(p0,i0,descr=arraydescr)
         jump(p0,i0)
         """
         vopt = self.vectoroptimizer(self.parse_loop(ops))
@@ -326,7 +331,7 @@
     def test_array_operation_indices_not_unrolled(self):
         ops = """
         [p0,i0]
-        raw_load(p0,i0,descr=arraydescr2)
+        raw_load(p0,i0,descr=arraydescr)
         jump(p0,i0)
         """
         vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),0)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -253,13 +253,12 @@
     def linear_find_smallest_type(self, loop):
         # O(#operations)
         for i,op in enumerate(loop.operations):
-            if op.is_raw_array_access():
+            if op.is_primitive_array_access():
                 descr = op.getdescr()
-                if not descr.is_array_of_pointers():
-                    byte_count = descr.get_item_size_in_bytes()
-                    if self.smallest_type_bytes == 0 \
-                       or byte_count < self.smallest_type_bytes:
-                        self.smallest_type_bytes = byte_count
+                byte_count = descr.get_item_size_in_bytes()
+                if self.smallest_type_bytes == 0 \
+                   or byte_count < self.smallest_type_bytes:
+                    self.smallest_type_bytes = byte_count
 
     def get_unroll_count(self, simd_vec_reg_bytes):
         """ This is an estimated number of further unrolls """
@@ -714,7 +713,7 @@
                 if origin_pack is None:
                     descr = lnode.getoperation().getdescr()
                     ptype = PackType.by_descr(descr, self.vec_reg_size)
-                    if lnode.getoperation().is_raw_load():
+                    if lnode.getoperation().is_primitive_load():
                         # load outputs value, no input
                         return Pair(lnode, rnode, None, ptype)
                     else:
@@ -757,7 +756,7 @@
         """ Blocks the packing of some operations """
         if inquestion.vector == -1:
             return True
-        if packed.is_raw_array_access():
+        if packed.is_primitive_array_access():
             if packed.getarg(1) == inquestion.result:
                 return True
         if not forward and inquestion.getopnum() == rop.INT_SIGNEXT:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -174,10 +174,19 @@
     def is_raw_array_access(self):
         return self.is_raw_load() or self.is_raw_store()
 
-    def is_raw_load(self):
+    def is_primitive_array_access(self):
+        """ Indicates that this operations loads/stores a
+        primitive type (int,float) """
+        if self.is_primitive_load() or self.is_primitive_store():
+            descr = self.getdescr()
+            if descr.is_array_of_primitives():
+                return True
+        return False
+
+    def is_primitive_load(self):
         return rop._RAW_LOAD_FIRST < self.getopnum() < rop._RAW_LOAD_LAST
 
-    def is_raw_store(self):
+    def is_primitive_store(self):
         return rop._RAW_STORE_FIRST < self.getopnum() < rop._RAW_STORE_LAST
 
     def is_comparison(self):
@@ -568,13 +577,13 @@
     #
     '_ALWAYS_PURE_LAST',  # ----- end of always_pure operations -----
 
+    '_RAW_LOAD_FIRST',
     'GETARRAYITEM_GC/2d',
-
-    '_RAW_LOAD_FIRST',
     'GETARRAYITEM_RAW/2d',
     'VEC_GETARRAYITEM_RAW/3d',
     'RAW_LOAD/2d',
     'VEC_RAW_LOAD/3d',
+    'VEC_GETARRAYITEM_GC/3d',
     '_RAW_LOAD_LAST',
 
     'GETINTERIORFIELD_GC/2d',
@@ -596,13 +605,14 @@
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'INCREMENT_DEBUG_COUNTER/1',
-    'SETARRAYITEM_GC/3d',
 
     '_RAW_STORE_FIRST',
+    'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
     'VEC_SETARRAYITEM_RAW/3d',
     'RAW_STORE/3d',
     'VEC_RAW_STORE/3d',
+    'VEC_SETARRAYITEM_GC/3d',
     '_RAW_STORE_LAST',
 
     'SETINTERIORFIELD_GC/3d',
@@ -796,8 +806,10 @@
 _opvector = {
     rop.RAW_LOAD:         rop.VEC_RAW_LOAD,
     rop.GETARRAYITEM_RAW: rop.VEC_GETARRAYITEM_RAW,
+    rop.GETARRAYITEM_GC: rop.VEC_GETARRAYITEM_GC,
     rop.RAW_STORE:        rop.VEC_RAW_STORE,
     rop.SETARRAYITEM_RAW: rop.VEC_SETARRAYITEM_RAW,
+    rop.SETARRAYITEM_GC: rop.VEC_SETARRAYITEM_GC,
 
     rop.INT_ADD:   rop.VEC_INT_ADD,
     rop.INT_SUB:   rop.VEC_INT_SUB,

From noreply at buildbot.pypy.org  Mon Aug 17 10:30:28 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 10:30:28 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-iterator-sharing: iterator sharing
 simplifications, removed the state list,
 indices for the iterator and moved the sharing logic in it's own function
Message-ID: <20150817083028.4BFB21C1370@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-iterator-sharing
Changeset: r79001:355ee4d5a425
Date: 2015-08-17 10:29 +0200
http://bitbucket.org/pypy/pypy/changeset/355ee4d5a425/

Log:	iterator sharing simplifications, removed the state list, indices
	for the iterator and moved the sharing logic in it's own function

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -34,61 +34,49 @@
     out_iter, out_state = out.create_iter(shape)
     shapelen = len(shape)
     res_dtype = out.get_dtype()
+    call2_func = try_to_share_iterators_call2(left_iter, right_iter,
+            left_state, right_state, out_state)
+    params = (space, shapelen, func, calc_dtype, res_dtype, out,
+              w_left, w_right, left_iter, right_iter, out_iter,
+              left_state, right_state, out_state)
+    return call2_func(*params)
 
-    states = [out_state,left_state,right_state]
-    left_index = 1
-    right_index = 2
-    # 1) get rid of scalar cases
-    # 2) switch through cases
-    # 3) you dont need states
-    # 4) no left_right_index
+def try_to_share_iterators_call2(left_iter, right_iter, left_state, right_state, out_state):
+    # these are all possible iterator sharing combinations
     # left == right == out
     # left == right
     # left == out
     # right == out
-    params = (space, shapelen, func, calc_dtype, res_dtype, out,
-              w_left, w_right, left_iter, right_iter, out_iter,
-              left_state, right_state, out_state)
-    if not right_iter:
-        # rhs is a scalar
-        del states[2]
-    else:
-        # rhs is NOT a scalar
+    right_out_equal = False
+    if right_iter:
+        # rhs is not a scalar
         if out_state.same(right_state):
-            # (1) out and right are the same -> remove right
-            right_index = 0
-            del states[2]
+            right_out_equal = True
     #
     if not left_iter:
         # lhs is a scalar
-        del states[1]
-        if right_index == 2:
-            right_index = 1
-            return call2_advance_out_right(*params)
+        if right_out_equal:
+            return call2_advance_out_left
+        else:
+            # left is a scalar, and right and out do not match
+            return call2_advance_out_left_right
     else:
         # lhs is NOT a scalar
         if out_state.same(left_state):
             # (2) out and left are the same -> remove left
-            left_index = 0
-            del states[1]
-            if right_index == 2:
-                right_index = 1
-            return call2_advance_out_right(*params)
+            if right_out_equal:
+                # the best case
+                return call2_advance_out
+            else:
+                return call2_advance_out_right
         else:
-            if len(states) == 3: # did not enter (1)
+            if right_out_equal:
+                return call2_advance_out_left
+            else:
                 if right_iter and right_state.same(left_state):
-                    right_index = 1
-                    del states[2]
-                    return call2_advance_out_left_eq_right(*params)
+                    return call2_advance_out_left_eq_right
                 else:
-                    # worst case
-                    return call2_advance_out_left_right(*params)
-            else:
-                return call2_advance_out_left(*params)
-
-    state_count = len(states)
-    if state_count == 1:
-        return call2_advance_out(*params)
+                    return call2_advance_out_left_right
 
     assert 0, "logical problem with the selection of the call 2 case"
 

From noreply at buildbot.pypy.org  Mon Aug 17 10:30:30 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 10:30:30 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: merged iterator sharing into the
	vecopt-merge
Message-ID: <20150817083030.A88131C1370@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79002:06ec92fa38c0
Date: 2015-08-17 10:30 +0200
http://bitbucket.org/pypy/pypy/changeset/06ec92fa38c0/

Log:	merged iterator sharing into the vecopt-merge

diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -83,6 +83,10 @@
         self._indices = indices
         self.offset = offset
 
+    def same(self, other):
+        if self.offset == other.offset:
+            return self.iterator.same_shape(other.iterator)
+        return False
 
 class ArrayIter(object):
     _immutable_fields_ = ['contiguous', 'array', 'size', 'ndim_m1', 'shape_m1[*]',
@@ -100,6 +104,7 @@
         self.array = array
         self.size = size
         self.ndim_m1 = len(shape) - 1
+        #
         self.shape_m1 = [s - 1 for s in shape]
         self.strides = strides
         self.backstrides = backstrides
@@ -113,6 +118,17 @@
                 factors[ndim-i-1] = factors[ndim-i] * shape[ndim-i]
         self.factors = factors
 
+    def same_shape(self, other):
+        """ if two iterators share the same shape,
+        next() only needs to be called on one!
+        """
+        return (self.contiguous == other.contiguous and
+                self.array.dtype is self.array.dtype and
+                self.shape_m1 == other.shape_m1 and
+                self.strides == other.strides and
+                self.backstrides == other.backstrides and
+                self.factors == other.factors)
+
     @jit.unroll_safe
     def reset(self, state=None, mutate=False):
         index = 0
@@ -196,7 +212,7 @@
         return state.index >= self.size
 
     def getitem(self, state):
-        assert state.iterator is self
+        # assert state.iterator is self
         return self.array.getitem(state.offset)
 
     def getitem_bool(self, state):
@@ -207,7 +223,6 @@
         assert state.iterator is self
         self.array.setitem(state.offset, elem)
 
-
 def AxisIter(array, shape, axis):
     strides = array.get_strides()
     backstrides = array.get_backstrides()
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -2,6 +2,7 @@
 operations. This is the place to look for all the computations that iterate
 over all the array elements.
 """
+import py
 from pypy.interpreter.error import OperationError
 from rpython.rlib import jit
 from rpython.rlib.rstring import StringBuilder
@@ -13,11 +14,6 @@
 from pypy.interpreter.argument import Arguments
 
 
-call2_driver = jit.JitDriver(
-    name='numpy_call2',
-    greens=['shapelen', 'func', 'left', 'right', 'calc_dtype', 'res_dtype'],
-    reds='auto', vectorize=True)
-
 def call2(space, shape, func, calc_dtype, w_lhs, w_rhs, out):
     if w_lhs.get_size() == 1:
         w_left = w_lhs.get_scalar_value().convert_to(space, calc_dtype)
@@ -38,28 +34,96 @@
     out_iter, out_state = out.create_iter(shape)
     shapelen = len(shape)
     res_dtype = out.get_dtype()
-    while not out_iter.done(out_state):
-        call2_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                     left=left_iter is None,
-                                     right=right_iter is None,
-                                     calc_dtype=calc_dtype, res_dtype=res_dtype)
-        if left_iter:
-            w_left = left_iter.getitem(left_state).convert_to(space, calc_dtype)
-            left_state = left_iter.next(left_state)
-        if right_iter:
-            w_right = right_iter.getitem(right_state).convert_to(space, calc_dtype)
-            right_state = right_iter.next(right_state)
-        w_out = func(calc_dtype, w_left, w_right)
-        out_iter.setitem(out_state, w_out.convert_to(space, res_dtype))
-        out_state = out_iter.next(out_state)
-        # if not set to None, the values will be loop carried
-        # (for the var,var case), forcing the vectorization to unpack
-        # the vector registers at the end of the loop
-        if left_iter:
-            w_left = None
-        if right_iter:
-            w_right = None
-    return out
+    call2_func = try_to_share_iterators_call2(left_iter, right_iter,
+            left_state, right_state, out_state)
+    params = (space, shapelen, func, calc_dtype, res_dtype, out,
+              w_left, w_right, left_iter, right_iter, out_iter,
+              left_state, right_state, out_state)
+    return call2_func(*params)
+
+def try_to_share_iterators_call2(left_iter, right_iter, left_state, right_state, out_state):
+    # these are all possible iterator sharing combinations
+    # left == right == out
+    # left == right
+    # left == out
+    # right == out
+    right_out_equal = False
+    if right_iter:
+        # rhs is not a scalar
+        if out_state.same(right_state):
+            right_out_equal = True
+    #
+    if not left_iter:
+        # lhs is a scalar
+        if right_out_equal:
+            return call2_advance_out_left
+        else:
+            # left is a scalar, and right and out do not match
+            return call2_advance_out_left_right
+    else:
+        # lhs is NOT a scalar
+        if out_state.same(left_state):
+            # (2) out and left are the same -> remove left
+            if right_out_equal:
+                # the best case
+                return call2_advance_out
+            else:
+                return call2_advance_out_right
+        else:
+            if right_out_equal:
+                return call2_advance_out_left
+            else:
+                if right_iter and right_state.same(left_state):
+                    return call2_advance_out_left_eq_right
+                else:
+                    return call2_advance_out_left_right
+
+    assert 0, "logical problem with the selection of the call 2 case"
+
+def generate_call2_cases(name, left_state, right_state):
+    call2_driver = jit.JitDriver(name='numpy_call2_' + name,
+        greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
+        reds='auto', vectorize=True)
+    #
+    advance_left_state = left_state == "left_state"
+    advance_right_state = right_state == "right_state"
+    code = """
+    def method(space, shapelen, func, calc_dtype, res_dtype, out,
+               w_left, w_right, left_iter, right_iter, out_iter,
+               left_state, right_state, out_state):
+        while not out_iter.done(out_state):
+            call2_driver.jit_merge_point(shapelen=shapelen, func=func,
+                    calc_dtype=calc_dtype, res_dtype=res_dtype)
+            if left_iter:
+                w_left = left_iter.getitem({left_state}).convert_to(space, calc_dtype)
+            if right_iter:
+                w_right = right_iter.getitem({right_state}).convert_to(space, calc_dtype)
+            w_out = func(calc_dtype, w_left, w_right)
+            out_iter.setitem(out_state, w_out.convert_to(space, res_dtype))
+            out_state = out_iter.next(out_state)
+            if advance_left_state and left_iter:
+                left_state = left_iter.next(left_state)
+            if advance_right_state and right_iter:
+                right_state = right_iter.next(right_state)
+            #
+            # if not set to None, the values will be loop carried
+            # (for the var,var case), forcing the vectorization to unpack
+            # the vector registers at the end of the loop
+            if left_iter:
+                w_left = None
+            if right_iter:
+                w_right = None
+        return out
+    """
+    exec(py.code.Source(code.format(left_state=left_state,right_state=right_state)).compile(), locals())
+    method.__name__ = "call2_" + name
+    return method
+
+call2_advance_out = generate_call2_cases("inc_out", "out_state", "out_state")
+call2_advance_out_left = generate_call2_cases("inc_out_left", "left_state", "out_state")
+call2_advance_out_right = generate_call2_cases("inc_out_right", "out_state", "right_state")
+call2_advance_out_left_eq_right = generate_call2_cases("inc_out_left_eq_right", "left_state", "left_state")
+call2_advance_out_left_right = generate_call2_cases("inc_out_left_right", "left_state", "right_state")
 
 call1_driver = jit.JitDriver(
     name='numpy_call1',
diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -911,8 +911,10 @@
     def test_multidim_slice(self):
         result = self.run('multidim_slice')
         assert result == 12
-        self.check_trace_count(2)
-        self.check_vectorized(1,0) # TODO?
+        self.check_trace_count(3)
+        # ::2 creates a view object -> needs an inner loop
+        # that iterates continous chunks of the matrix
+        self.check_vectorized(1,1) 
 
     # NOT WORKING
 
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -155,6 +155,13 @@
     def __repr__(self):
         return 'ArrayDescr(%r)' % (self.OUTERA,)
 
+    def is_array_of_primitives(self):
+        kind = getkind(self.A.OF)
+        return kind == 'float' or \
+               kind == 'int' or \
+               kind == ''
+
+
     def is_array_of_pointers(self):
         return getkind(self.A.OF) == 'ref'
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -203,6 +203,11 @@
     def getconcrete_type(self):
         return self.concrete_type
 
+    def is_array_of_primitives(self):
+        return self.flag == FLAG_FLOAT or \
+               self.flag == FLAG_SIGNED or \
+               self.flag == FLAG_UNSIGNED
+
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
 
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -344,6 +344,8 @@
                          rop.VEC_RAW_STORE,
                          rop.VEC_GETARRAYITEM_RAW,
                          rop.VEC_SETARRAYITEM_RAW,
+                         rop.VEC_GETARRAYITEM_GC,
+                         rop.VEC_SETARRAYITEM_GC,
                          ):      # list of opcodes never executed by pyjitpl
                 continue
             if rop._VEC_PURE_FIRST <= value <= rop._VEC_PURE_LAST:
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -805,8 +805,9 @@
     def operation_{name}(self, op, node):
         descr = op.getdescr()
         idx_ref = self.get_or_create(op.getarg(1))
-        node.memory_ref = MemoryRef(op, idx_ref, {raw_access})
-        self.memory_refs[node] = node.memory_ref
+        if descr.is_array_of_primitives():
+            node.memory_ref = MemoryRef(op, idx_ref, {raw_access})
+            self.memory_refs[node] = node.memory_ref
     """
     exec py.code.Source(array_access_source
            .format(name='RAW_LOAD',raw_access=True)).compile()
@@ -816,6 +817,10 @@
            .format(name='GETARRAYITEM_RAW',raw_access=False)).compile()
     exec py.code.Source(array_access_source
            .format(name='SETARRAYITEM_RAW',raw_access=False)).compile()
+    exec py.code.Source(array_access_source
+           .format(name='GETARRAYITEM_GC',raw_access=False)).compile()
+    exec py.code.Source(array_access_source
+           .format(name='SETARRAYITEM_GC',raw_access=False)).compile()
     del array_access_source
 integral_dispatch_opt = make_dispatcher_method(IntegralForwardModification, 'operation_')
 IntegralForwardModification.inspect_operation = integral_dispatch_opt
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -692,8 +692,10 @@
 
     rop.VEC_RAW_LOAD:         LOAD_TRANS,
     rop.VEC_GETARRAYITEM_RAW: LOAD_TRANS,
+    rop.VEC_GETARRAYITEM_GC: LOAD_TRANS,
     rop.VEC_RAW_STORE:        STORE_TRANS,
     rop.VEC_SETARRAYITEM_RAW: STORE_TRANS,
+    rop.VEC_SETARRAYITEM_GC: STORE_TRANS,
 
     rop.VEC_CAST_FLOAT_TO_SINGLEFLOAT: OpToVectorOpConv(PT_DOUBLE_2, PT_FLOAT_2),
     rop.VEC_CAST_SINGLEFLOAT_TO_FLOAT: OpToVectorOpConv(PT_FLOAT_2, PT_DOUBLE_2),
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -212,21 +212,6 @@
         """
         self.assert_vectorize(self.parse_loop(ops), self.parse_loop(ops))
 
-    def test_vectorize_skip_impossible_2(self):
-        ops = """
-        [p0,i0]
-        i1 = int_add(i0,1)
-        i2 = int_le(i1, 10)
-        guard_true(i2) []
-        i3 = getarrayitem_gc(p0,i0,descr=intarraydescr)
-        jump(p0,i1)
-        """
-        try:
-            self.vectorize(self.parse_loop(ops))
-            py.test.fail("should not happend")
-        except NotAVectorizeableLoop:
-            pass
-
     def test_unroll_empty_stays_empty(self):
         """ has no operations in this trace, thus it stays empty
         after unrolling it 2 times """
@@ -264,6 +249,26 @@
         """
         self.assert_vectorize(self.parse_loop(ops), self.parse_loop(ops))
 
+    def test_load_primitive_python_list(self):
+        """ it currently rejects pointer arrays """
+        ops = """
+        [p0,i0]
+        i2 = getarrayitem_gc(p0,i0,descr=floatarraydescr)
+        i1 = int_add(i0,1)
+        i3 = getarrayitem_gc(p0,i1,descr=floatarraydescr)
+        i4 = int_add(i1,1)
+        jump(p0,i4)
+        """
+        opt = """
+        [p0,i0]
+        i1 = int_add(i0,1)
+        i2 = int_add(i0,2)
+        i3 = vec_getarrayitem_gc(p0,i0,2,descr=floatarraydescr)
+        jump(p0,i2)
+        """
+        vopt = self.vectorize(self.parse_loop(ops),0)
+        self.assert_equal(vopt.loop, self.parse_loop(opt))
+
     def test_vect_unroll_char(self):
         """ a 16 byte vector register can hold 16 bytes thus 
         it is unrolled 16 times. (it is the smallest type in the trace) """
@@ -316,7 +321,7 @@
     def test_estimate_unroll_factor_smallest_byte_zero(self):
         ops = """
         [p0,i0]
-        raw_load(p0,i0,descr=arraydescr2)
+        raw_load(p0,i0,descr=arraydescr)
         jump(p0,i0)
         """
         vopt = self.vectoroptimizer(self.parse_loop(ops))
@@ -326,7 +331,7 @@
     def test_array_operation_indices_not_unrolled(self):
         ops = """
         [p0,i0]
-        raw_load(p0,i0,descr=arraydescr2)
+        raw_load(p0,i0,descr=arraydescr)
         jump(p0,i0)
         """
         vopt = self.vectoroptimizer_unrolled(self.parse_loop(ops),0)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -253,13 +253,12 @@
     def linear_find_smallest_type(self, loop):
         # O(#operations)
         for i,op in enumerate(loop.operations):
-            if op.is_raw_array_access():
+            if op.is_primitive_array_access():
                 descr = op.getdescr()
-                if not descr.is_array_of_pointers():
-                    byte_count = descr.get_item_size_in_bytes()
-                    if self.smallest_type_bytes == 0 \
-                       or byte_count < self.smallest_type_bytes:
-                        self.smallest_type_bytes = byte_count
+                byte_count = descr.get_item_size_in_bytes()
+                if self.smallest_type_bytes == 0 \
+                   or byte_count < self.smallest_type_bytes:
+                    self.smallest_type_bytes = byte_count
 
     def get_unroll_count(self, simd_vec_reg_bytes):
         """ This is an estimated number of further unrolls """
@@ -667,7 +666,7 @@
                 if origin_pack is None:
                     descr = lnode.getoperation().getdescr()
                     ptype = PackType.by_descr(descr, self.vec_reg_size)
-                    if lnode.getoperation().is_raw_load():
+                    if lnode.getoperation().is_primitive_load():
                         # load outputs value, no input
                         return Pair(lnode, rnode, None, ptype)
                     else:
@@ -710,7 +709,7 @@
         """ Blocks the packing of some operations """
         if inquestion.vector == -1:
             return True
-        if packed.is_raw_array_access():
+        if packed.is_primitive_array_access():
             if packed.getarg(1) == inquestion.result:
                 return True
         if not forward and inquestion.getopnum() == rop.INT_SIGNEXT:
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -174,10 +174,19 @@
     def is_raw_array_access(self):
         return self.is_raw_load() or self.is_raw_store()
 
-    def is_raw_load(self):
+    def is_primitive_array_access(self):
+        """ Indicates that this operations loads/stores a
+        primitive type (int,float) """
+        if self.is_primitive_load() or self.is_primitive_store():
+            descr = self.getdescr()
+            if descr.is_array_of_primitives():
+                return True
+        return False
+
+    def is_primitive_load(self):
         return rop._RAW_LOAD_FIRST < self.getopnum() < rop._RAW_LOAD_LAST
 
-    def is_raw_store(self):
+    def is_primitive_store(self):
         return rop._RAW_STORE_FIRST < self.getopnum() < rop._RAW_STORE_LAST
 
     def is_comparison(self):
@@ -568,13 +577,13 @@
     #
     '_ALWAYS_PURE_LAST',  # ----- end of always_pure operations -----
 
+    '_RAW_LOAD_FIRST',
     'GETARRAYITEM_GC/2d',
-
-    '_RAW_LOAD_FIRST',
     'GETARRAYITEM_RAW/2d',
     'VEC_GETARRAYITEM_RAW/3d',
     'RAW_LOAD/2d',
     'VEC_RAW_LOAD/3d',
+    'VEC_GETARRAYITEM_GC/3d',
     '_RAW_LOAD_LAST',
 
     'GETINTERIORFIELD_GC/2d',
@@ -596,13 +605,14 @@
     '_NOSIDEEFFECT_LAST', # ----- end of no_side_effect operations -----
 
     'INCREMENT_DEBUG_COUNTER/1',
-    'SETARRAYITEM_GC/3d',
 
     '_RAW_STORE_FIRST',
+    'SETARRAYITEM_GC/3d',
     'SETARRAYITEM_RAW/3d',
     'VEC_SETARRAYITEM_RAW/3d',
     'RAW_STORE/3d',
     'VEC_RAW_STORE/3d',
+    'VEC_SETARRAYITEM_GC/3d',
     '_RAW_STORE_LAST',
 
     'SETINTERIORFIELD_GC/3d',
@@ -796,8 +806,10 @@
 _opvector = {
     rop.RAW_LOAD:         rop.VEC_RAW_LOAD,
     rop.GETARRAYITEM_RAW: rop.VEC_GETARRAYITEM_RAW,
+    rop.GETARRAYITEM_GC: rop.VEC_GETARRAYITEM_GC,
     rop.RAW_STORE:        rop.VEC_RAW_STORE,
     rop.SETARRAYITEM_RAW: rop.VEC_SETARRAYITEM_RAW,
+    rop.SETARRAYITEM_GC: rop.VEC_SETARRAYITEM_GC,
 
     rop.INT_ADD:   rop.VEC_INT_ADD,
     rop.INT_SUB:   rop.VEC_INT_SUB,

From noreply at buildbot.pypy.org  Mon Aug 17 10:33:54 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 10:33:54 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-iterator-sharing: merged into
	vecopt-merge
Message-ID: <20150817083354.8429C1C1370@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-iterator-sharing
Changeset: r79003:ee4fde461c8b
Date: 2015-08-17 10:33 +0200
http://bitbucket.org/pypy/pypy/changeset/ee4fde461c8b/

Log:	merged into vecopt-merge


From noreply at buildbot.pypy.org  Mon Aug 17 10:35:44 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 10:35:44 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt2: old version of vecopt2
Message-ID: <20150817083544.EF6081C1370@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt2
Changeset: r79004:e9e3e4efdb25
Date: 2015-08-17 10:35 +0200
http://bitbucket.org/pypy/pypy/changeset/e9e3e4efdb25/

Log:	old version of vecopt2


From noreply at buildbot.pypy.org  Mon Aug 17 10:47:40 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 10:47:40 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: fix for wrong
	assumption
Message-ID: <20150817084740.A2F5C1C12F4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1933:2f5c2406cf18
Date: 2015-08-11 14:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/2f5c2406cf18/

Log:	fix for wrong assumption

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1821,12 +1821,10 @@
             ssize_t size = (ssize_t)lst->items[i+1];
 
             if (start + size == obj_start) {
-                /* merge!
-                   Note: we cannot overlap pages by adding to a range
-                   in this way, since the first slot in a smallmalloc
-                   page is always unused. Thus, we never merge the
-                   last obj in a page with the first obj in a successor
-                   page. */
+                /* merge! */
+                if ((size + obj_size) + ((uintptr_t)start & 4095) > 4096)
+                    break;      /* doesn't fit inside the same page */
+
                 lst->items[i+1] = size + obj_size;
                 return;
             }
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -113,6 +113,8 @@
        pages) */
     /* XXX: not much different from before. Maybe try a ranges list
        per size class. */
+    /* XXX: also, we could sweep these ranges on abort and thereby
+       free these overflow objs early */
     struct list_s *small_overflow_obj_ranges;
 
     uint8_t privatization_lock;  // XXX KILL

From noreply at buildbot.pypy.org  Mon Aug 17 10:47:42 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 10:47:42 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: actually yes,
	quadratic is a problem :)
Message-ID: <20150817084742.A4F781C12F4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1934:bbc6c59a805c
Date: 2015-08-11 15:06 +0200
http://bitbucket.org/pypy/stmgc/changeset/bbc6c59a805c/

Log:	actually yes, quadratic is a problem :)

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1813,10 +1813,11 @@
 
     struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges;
     if (!list_is_empty(lst)) {
-        /* try to merge with other ranges (XXX: quadratic, problem?) */
+        /* seems to not help to look for merges in this way: */
         stm_char *obj_start = (stm_char*)obj;
         long i;
-        for (i = lst->count - 2; i >= 0; i -= 2) {
+        long min = lst->count - 4 * 2; /* go back 4 elems */
+        for (i = lst->count - 2; i >= min; i -= 2) {
             stm_char *start = (stm_char*)lst->items[i];
             ssize_t size = (ssize_t)lst->items[i+1];
 

From noreply at buildbot.pypy.org  Mon Aug 17 10:47:44 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 10:47:44 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: use separate list per
 size class of obj to increase probability of merges
Message-ID: <20150817084744.961811C12F4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1935:9339deb15d43
Date: 2015-08-11 15:49 +0200
http://bitbucket.org/pypy/stmgc/changeset/9339deb15d43/

Log:	use separate list per size class of obj to increase probability of
	merges

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1155,7 +1155,10 @@
 
     assert(list_is_empty(STM_PSEGMENT->modified_old_objects));
     assert(list_is_empty(STM_PSEGMENT->large_overflow_objects));
-    assert(list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges));
+#ifndef NDEBUG
+    for (long i = 2; i < GC_N_SMALL_REQUESTS; i++)
+        assert(list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges[i]));
+#endif
     assert(list_is_empty(STM_PSEGMENT->objects_pointing_to_nursery));
     assert(list_is_empty(STM_PSEGMENT->young_weakrefs));
     assert(tree_is_cleared(STM_PSEGMENT->young_outside_nursery));
@@ -1244,7 +1247,9 @@
     list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
     list_clear(STM_PSEGMENT->old_objects_with_cards_set);
     list_clear(STM_PSEGMENT->large_overflow_objects);
-    list_clear(STM_PSEGMENT->small_overflow_obj_ranges);
+    for (long i = 2; i < GC_N_SMALL_REQUESTS; i++)
+        list_clear(STM_PSEGMENT->small_overflow_obj_ranges[i]);
+
     if (tl != NULL)
         timing_event(tl, event);
 
@@ -1288,18 +1293,20 @@
         release_privatization_lock(STM_SEGMENT->segment_num);
     }
 
-    if (!list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges)) {
-        acquire_privatization_lock(STM_SEGMENT->segment_num);
+    for (long i = 2; i < GC_N_SMALL_REQUESTS; i++) {
+        if (!list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges[i])) {
+            acquire_privatization_lock(STM_SEGMENT->segment_num);
 
-        struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges;
-        while (!list_is_empty(lst)) {
-            ssize_t len = (ssize_t)list_pop_item(lst);
-            stm_char *start = (stm_char*)list_pop_item(lst);
-            _synchronize_fragment(start, len);
+            struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges[i];
+            while (!list_is_empty(lst)) {
+                ssize_t len = (ssize_t)list_pop_item(lst);
+                stm_char *start = (stm_char*)list_pop_item(lst);
+                _synchronize_fragment(start, len);
+            }
+
+            synchronize_objects_flush();
+            release_privatization_lock(STM_SEGMENT->segment_num);
         }
-
-        synchronize_objects_flush();
-        release_privatization_lock(STM_SEGMENT->segment_num);
     }
 
     /* we can as well clear the list here, since the
@@ -1311,7 +1318,10 @@
        unknown-to-the-segment/uncommitted things.
     */
     list_clear(STM_PSEGMENT->large_overflow_objects);
-    list_clear(STM_PSEGMENT->small_overflow_obj_ranges);
+#ifndef NDEBUG
+    for (long i = 2; i < GC_N_SMALL_REQUESTS; i++)
+        assert(list_is_empty(STM_PSEGMENT->small_overflow_obj_ranges[i]));
+#endif
 }
 
 
@@ -1532,7 +1542,8 @@
     list_clear(pseg->objects_pointing_to_nursery);
     list_clear(pseg->old_objects_with_cards_set);
     list_clear(pseg->large_overflow_objects);
-    list_clear(pseg->small_overflow_obj_ranges);
+    for (long i = 2; i < GC_N_SMALL_REQUESTS; i++)
+        list_clear(pseg->small_overflow_obj_ranges[i]);
     list_clear(pseg->young_weakrefs);
 #pragma pop_macro("STM_SEGMENT")
 #pragma pop_macro("STM_PSEGMENT")
@@ -1811,12 +1822,12 @@
         (struct object_s *)REAL_ADDRESS(STM_SEGMENT->segment_base, obj));
     OPT_ASSERT(obj_size >= 16);
 
-    struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges;
+    struct list_s *lst = STM_PSEGMENT->small_overflow_obj_ranges[obj_size / 8];
     if (!list_is_empty(lst)) {
         /* seems to not help to look for merges in this way: */
         stm_char *obj_start = (stm_char*)obj;
         long i;
-        long min = lst->count - 4 * 2; /* go back 4 elems */
+        long min = lst->count - 2 * 2; /* go back 4 elems */
         for (i = lst->count - 2; i >= min; i -= 2) {
             stm_char *start = (stm_char*)lst->items[i];
             ssize_t size = (ssize_t)lst->items[i+1];
@@ -1833,6 +1844,6 @@
     }
 
     /* no merge was found */
-    STM_PSEGMENT->small_overflow_obj_ranges =
+    STM_PSEGMENT->small_overflow_obj_ranges[obj_size / 8] =
         list_append2(lst, (uintptr_t)obj, (uintptr_t)obj_size);
 }
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -111,11 +111,9 @@
        need to be flushed to other segments on commit (like
        large_overflow_objects). (unsorted, a range never overlaps
        pages) */
-    /* XXX: not much different from before. Maybe try a ranges list
-       per size class. */
     /* XXX: also, we could sweep these ranges on abort and thereby
        free these overflow objs early */
-    struct list_s *small_overflow_obj_ranges;
+    struct list_s *small_overflow_obj_ranges[GC_N_SMALL_REQUESTS];
 
     uint8_t privatization_lock;  // XXX KILL
 
diff --git a/c8/stm/forksupport.c b/c8/stm/forksupport.c
--- a/c8/stm/forksupport.c
+++ b/c8/stm/forksupport.c
@@ -98,7 +98,8 @@
         STM_PSEGMENT->safe_point = SP_NO_TRANSACTION;
         list_clear(STM_PSEGMENT->objects_pointing_to_nursery);
         list_clear(STM_PSEGMENT->large_overflow_objects);
-        list_clear(STM_PSEGMENT->small_overflow_obj_ranges);
+        for (long k = 2; k < GC_N_SMALL_REQUESTS; k++)
+            list_clear(STM_PSEGMENT->small_overflow_obj_ranges[k]);
 
         s_mutex_unlock();
         return;
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -599,43 +599,40 @@
 
         /* fix ranges in small_overflow_obj_ranges that may have
            holes from dying objs */
-        lst = pseg->small_overflow_obj_ranges;
-        if (!list_is_empty(lst)) {
-            long j;
-            struct list_s *new_lst = list_create();
-            for (j = lst->count - 2; j >= 0; j -= 2) {
-                stm_char *start = (stm_char*)lst->items[j];
-                ssize_t size = (ssize_t)lst->items[j+1];
+        for (long k = 2; k < GC_N_SMALL_REQUESTS; k++) {
+            lst = pseg->small_overflow_obj_ranges[k];
+            if (!list_is_empty(lst)) {
+                long j;
+                struct list_s *new_lst = list_create();
+                for (j = lst->count - 2; j >= 0; j -= 2) {
+                    stm_char *start = (stm_char*)lst->items[j];
+                    ssize_t size = (ssize_t)lst->items[j+1];
+                    size_t obj_size = k * 8;
 
-                struct object_s *realobj = (struct object_s *)
-                    REAL_ADDRESS(pseg->pub.segment_base, start);
-                size_t obj_size = stmcb_size_rounded_up(realobj);
-                assert(size % obj_size == 0);
-
-                stm_char *obj = start, *range_end = start + size;
-                for (; obj < range_end; obj += obj_size) {
-                    /* note: all these obj-slots *are* in use by overflow objs */
-                    if (!mark_visited_test((object_t*)obj)) {
-                        if (obj != start) {
-                            new_lst = list_append2(new_lst,
-                                                   (uintptr_t)start,
-                                                   (uintptr_t)(obj - start));
+                    stm_char *obj = start, *range_end = start + size;
+                    for (; obj < range_end; obj += obj_size) {
+                        /* note: all these obj-slots *are* in use by overflow objs */
+                        if (!mark_visited_test((object_t*)obj)) {
+                            if (obj != start) {
+                                new_lst = list_append2(new_lst,
+                                                       (uintptr_t)start,
+                                                       (uintptr_t)(obj - start));
+                            }
+                            start = obj + obj_size; // next start
                         }
-                        start = obj + obj_size; // next start
+                    }
+                    if (obj != start) {
+                        new_lst = list_append2(new_lst,
+                                               (uintptr_t)start,
+                                               (uintptr_t)(obj - start));
                     }
                 }
-                if (obj != start) {
-                    new_lst = list_append2(new_lst,
-                                           (uintptr_t)start,
-                                           (uintptr_t)(obj - start));
-                }
+
+                list_free(lst);
+                pseg->small_overflow_obj_ranges[k] = new_lst;
             }
-
-            list_free(lst);
-            pseg->small_overflow_obj_ranges = new_lst;
         }
 
-
         /* Remove from 'modified_old_objects' all old hashtables that die */
         {
             lst = pseg->modified_old_objects;
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -101,7 +101,8 @@
         pr->pub.segment_base = segment_base;
         pr->modified_old_objects = list_create();
         pr->large_overflow_objects = list_create();
-        pr->small_overflow_obj_ranges = list_create();
+        for (long k = 2; k < GC_N_SMALL_REQUESTS; k++)
+            pr->small_overflow_obj_ranges[k] = list_create();
         pr->young_weakrefs = list_create();
         pr->old_weakrefs = list_create();
         pr->objects_pointing_to_nursery = list_create();
@@ -159,9 +160,13 @@
         list_free(pr->old_objects_with_cards_set);
         list_free(pr->modified_old_objects);
         assert(list_is_empty(pr->large_overflow_objects));
-        assert(list_is_empty(pr->small_overflow_obj_ranges));
+#ifndef NDEBUG
+        for (long k = 2; k < GC_N_SMALL_REQUESTS; k++)
+            assert(list_is_empty(pr->small_overflow_obj_ranges[k]));
+#endif
         list_free(pr->large_overflow_objects);
-        list_free(pr->small_overflow_obj_ranges);
+        for (long k = 2; k < GC_N_SMALL_REQUESTS; k++)
+            list_free(pr->small_overflow_obj_ranges[k]);
         list_free(pr->young_weakrefs);
         list_free(pr->old_weakrefs);
         tree_free(pr->young_outside_nursery);

From noreply at buildbot.pypy.org  Mon Aug 17 10:47:46 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 10:47:46 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: fix;
	but anyway seems to go nowhere
Message-ID: <20150817084746.A36BE1C12F4@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1936:3dea513c6824
Date: 2015-08-17 10:51 +0200
http://bitbucket.org/pypy/stmgc/changeset/3dea513c6824/

Log:	fix; but anyway seems to go nowhere

	Merging ranges in small_overflow_obj_ranges_add was basically the
	only way this branch could improve sync-time. However, even with the
	quadratic, dead-slow version of merging, only 8% of objs passed to
	this function allowed a merge to happen (and the current version
	reaches ~7%). In this light, the branch is probably useless. It also
	suggests that smallmalloc often allocates in isolated slots of
	already-used pages. Meaning, if the allocation pattern of
	smallmalloc does not change, there is little chance of syncing
	continuous memory ranges instead of individual objects.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1827,8 +1827,8 @@
         /* seems to not help to look for merges in this way: */
         stm_char *obj_start = (stm_char*)obj;
         long i;
-        long min = lst->count - 2 * 2; /* go back 4 elems */
-        for (i = lst->count - 2; i >= min; i -= 2) {
+        long min = lst->count - 4 * 2; /* go back 4 elems */
+        for (i = lst->count - 2; i >= min && i >= 0; i -= 2) {
             stm_char *start = (stm_char*)lst->items[i];
             ssize_t size = (ssize_t)lst->items[i+1];
 

From noreply at buildbot.pypy.org  Mon Aug 17 11:19:49 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 11:19:49 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: added some comments to the state
	switching
Message-ID: <20150817091949.6BD911C1334@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79005:299b91377f5f
Date: 2015-08-17 10:40 +0200
http://bitbucket.org/pypy/pypy/changeset/299b91377f5f/

Log:	added some comments to the state switching

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -58,7 +58,7 @@
         if right_out_equal:
             return call2_advance_out_left
         else:
-            # left is a scalar, and right and out do not match
+            # worst case, nothing can be shared and lhs is a scalar
             return call2_advance_out_left_right
     else:
         # lhs is NOT a scalar
@@ -71,14 +71,17 @@
                 return call2_advance_out_right
         else:
             if right_out_equal:
+                # right and out are equal, only advance left and out
                 return call2_advance_out_left
             else:
                 if right_iter and right_state.same(left_state):
+                    # left and right are equal, but still need to advance out
                     return call2_advance_out_left_eq_right
                 else:
+                    # worst case, nothing can be shared
                     return call2_advance_out_left_right
 
-    assert 0, "logical problem with the selection of the call 2 case"
+    assert 0, "logical problem with the selection of the call2 case"
 
 def generate_call2_cases(name, left_state, right_state):
     call2_driver = jit.JitDriver(name='numpy_call2_' + name,

From noreply at buildbot.pypy.org  Mon Aug 17 11:19:51 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 11:19:51 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: added iterator sharing to call1
 (only one possibility which makes things easier)
Message-ID: <20150817091951.867CB1C1334@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79006:a610dc93d3a2
Date: 2015-08-17 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/a610dc93d3a2/

Log:	added iterator sharing to call1 (only one possibility which makes
	things easier)

diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -130,7 +130,7 @@
 
 call1_driver = jit.JitDriver(
     name='numpy_call1',
-    greens=['shapelen', 'func', 'calc_dtype', 'res_dtype'],
+    greens=['shapelen', 'share_iterator', 'func', 'calc_dtype', 'res_dtype'],
     reds='auto', vectorize=True)
 
 def call1(space, shape, func, calc_dtype, w_obj, w_ret):
@@ -139,13 +139,24 @@
     out_iter, out_state = w_ret.create_iter(shape)
     shapelen = len(shape)
     res_dtype = w_ret.get_dtype()
+    share_iterator = out_state.same(obj_state)
     while not out_iter.done(out_state):
         call1_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                     share_iterator=share_iterator,
                                      calc_dtype=calc_dtype, res_dtype=res_dtype)
-        elem = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
+        if share_iterator:
+            # use out state as param to getitem
+            elem = obj_iter.getitem(out_state).convert_to(space, calc_dtype)
+        else:
+            elem = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, func(calc_dtype, elem).convert_to(space, res_dtype))
-        out_state = out_iter.next(out_state)
-        obj_state = obj_iter.next(obj_state)
+        if share_iterator:
+            # only advance out, they share the same iteration space
+            out_state = out_iter.next(out_state)
+        else:
+            out_state = out_iter.next(out_state)
+            obj_state = obj_iter.next(obj_state)
+        elem = None
     return w_ret
 
 call_many_to_one_driver = jit.JitDriver(

From noreply at buildbot.pypy.org  Mon Aug 17 11:19:53 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 11:19:53 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: moved post/pre unpacking from
 vectorize.py into schedule.py,
 this removes unnecessary object allocations for each scheduling step,
 makes things more readable
Message-ID: <20150817091953.9AA721C1334@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79007:318a57aaddb8
Date: 2015-08-17 11:18 +0200
http://bitbucket.org/pypy/pypy/changeset/318a57aaddb8/

Log:	moved post/pre unpacking from vectorize.py into schedule.py, this
	removes unnecessary object allocations for each scheduling step,
	makes things more readable

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -244,7 +244,7 @@
         self.output_type = None
         self.costmodel = None
 
-    def as_vector_operation(self, pack, sched_data, oplist):
+    def as_vector_operation(self, pack, sched_data, scheduler, oplist):
         self.sched_data = sched_data
         self.vecops = oplist
         self.costmodel = sched_data.costmodel
@@ -257,7 +257,9 @@
         #
         if self.must_be_full_but_is_not(pack):
             for op in pack.operations:
-                self.vecops.append(op.getoperation())
+                operation = op.getoperation()
+                self.sched_data.unpack_from_vector(operation, scheduler)
+                self.vecops.append(operation)
         else:
             self.pack = pack
             self.transform_pack()
@@ -757,15 +759,20 @@
         renamer = scheduler.renamer
         if candidate.pack:
             for node in candidate.pack.operations:
-                self.unpack_from_vector(candidate.getoperation(), renamer)
+                renamer.rename(node.getoperation())
                 scheduler.scheduled(node)
-                #renamer.rename(node.getoperation())
             self.as_vector_operation(scheduler, candidate.pack)
         else:
-            self.unpack_from_vector(candidate.getoperation(), renamer)
+            op = candidate.getoperation()
+            renamer.rename(op)
+            self.unpack_from_vector(op, scheduler)
             scheduler.scheduled(candidate)
-            #renamer.rename(candidate.getoperation())
             op = candidate.getoperation()
+            #
+            # prevent some instructions in the resulting trace!
+            if op.getopnum() in (rop.DEBUG_MERGE_POINT,
+                                 rop.GUARD_EARLY_EXIT):
+                return
             scheduler.oplist.append(op)
 
     def as_vector_operation(self, scheduler, pack):
@@ -777,7 +784,7 @@
         oplist = scheduler.oplist
         position = len(oplist)
         op = pack.operations[0].getoperation()
-        determine_trans(op).as_vector_operation(pack, self, oplist)
+        determine_trans(op).as_vector_operation(pack, self, scheduler, oplist)
         #
         # XXX
         if pack.is_accumulating():
@@ -786,16 +793,15 @@
             for node in pack.operations:
                 op = node.getoperation()
                 assert op.result is not None
-                preproc_renamer.start_renaming(op.result, box)
+                scheduler.renamer.start_renaming(op.result, box)
 
-    def unpack_from_vector(self, op, renamer):
-        renamer.rename(op)
+    def unpack_from_vector(self, op, scheduler):
         args = op.getarglist()
 
         # unpack for an immediate use
         for i, arg in enumerate(op.getarglist()):
             if isinstance(arg, Box):
-                argument = self._unpack_from_vector(i, arg, renamer)
+                argument = self._unpack_from_vector(i, arg, scheduler)
                 if arg is not argument:
                     op.setarg(i, argument)
         if op.result:
@@ -805,11 +811,11 @@
             fail_args = op.getfailargs()
             for i, arg in enumerate(fail_args):
                 if arg and isinstance(arg, Box):
-                    argument = self._unpack_from_vector(i, arg, renamer)
+                    argument = self._unpack_from_vector(i, arg, scheduler)
                     if arg is not argument:
                         fail_args[i] = argument
 
-    def _unpack_from_vector(self, i, arg, renamer):
+    def _unpack_from_vector(self, i, arg, scheduler):
         if arg in self.seen or arg.type == 'V':
             return arg
         (j, vbox) = self.getvector_of_box(arg)
@@ -818,14 +824,14 @@
                 return arg
             arg_cloned = arg.clonebox()
             self.seen[arg_cloned] = None
-            renamer.start_renaming(arg, arg_cloned)
+            scheduler.renamer.start_renaming(arg, arg_cloned)
             self.setvector_of_box(arg_cloned, j, vbox)
             cj = ConstInt(j)
             ci = ConstInt(1)
             opnum = getunpackopnum(vbox.gettype())
             unpack_op = ResOperation(opnum, [vbox, cj, ci], arg_cloned)
             self.costmodel.record_vector_unpack(vbox, j, 1)
-            self.emit_operation(unpack_op)
+            scheduler.oplist.append(unpack_op)
             return arg_cloned
         return arg
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -144,14 +144,6 @@
         if not self.costmodel.profitable():
             raise NotAProfitableLoop()
 
-    def emit_operation(self, op):
-        if op.getopnum() == rop.DEBUG_MERGE_POINT:
-            return
-        if op.getopnum() == rop.GUARD_EARLY_EXIT:
-            return
-        self._last_emitted_op = op
-        self._newoperations.append(op)
-
     def emit_unrolled_operation(self, op):
         self._last_emitted_op = op
         self._newoperations.append(op)

From noreply at buildbot.pypy.org  Mon Aug 17 11:19:55 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 17 Aug 2015 11:19:55 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge-opt: closing,
	work was continued in vecopt-merge
Message-ID: <20150817091955.8E7551C1334@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge-opt
Changeset: r79008:b414e83c5a5f
Date: 2015-08-17 11:19 +0200
http://bitbucket.org/pypy/pypy/changeset/b414e83c5a5f/

Log:	closing, work was continued in vecopt-merge


From noreply at buildbot.pypy.org  Mon Aug 17 14:15:35 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Raise struct.error instead of
 OverflowError if someone passes a larger-than-machine-size integer to
 struct.pack().
Message-ID: <20150817121535.0E0B91C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: 
Changeset: r79009:06d07916e1be
Date: 2015-08-17 02:22 +0200
http://bitbucket.org/pypy/pypy/changeset/06d07916e1be/

Log:	Raise struct.error instead of OverflowError if someone passes a
	larger-than-machine-size integer to struct.pack().

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -82,7 +82,13 @@
                 w_index = space.int(w_obj)   # wrapped float -> wrapped int or long
             if w_index is None:
                 raise StructError("cannot convert argument to integer")
-        return getattr(space, meth)(w_index)
+        method = getattr(space, meth)
+        try:
+            return method(w_index)
+        except OperationError as e:
+            if e.match(self.space, self.space.w_OverflowError):
+                raise StructError("argument out of range")
+            raise
 
     def accept_bool_arg(self):
         w_obj = self.accept_obj_arg()
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -428,6 +428,9 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_overflow(self):
+        raises(self.struct.error, self.struct.pack, 'i', 1<<65)
+
 
 class AppTestStructBuffer(object):
     spaceconfig = dict(usemodules=['struct', '__pypy__'])

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:38 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150817121538.E20FA1C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79010:81d2a303c872
Date: 2015-08-17 02:41 +0200
http://bitbucket.org/pypy/pypy/changeset/81d2a303c872/

Log:	hg merge default

diff too long, truncating to 2000 out of 5812 lines

diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("= 2.6.1,
+   you get a new globals dictionary for every call (but then, all globals
+   dictionaries are all kept alive forever, in ``sys._pypy_execute_source``).
 
 .. function:: int pypy_execute_source_ptr(char* source, void* ptr);
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -55,3 +55,15 @@
 .. branch: nditer-revisited
 
 Implement nditer 'buffered' flag and fix some edge cases
+
+.. branch: ufunc-reduce
+
+Allow multiple axes in ufunc.reduce()
+
+.. branch: fix-tinylang-goals
+
+Update tinylang goals to match current rpython
+
+.. branch: vmprof-review
+
+Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -135,13 +135,7 @@
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
-        after = rffi.aroundstate.after
-        if after: after()
-        source = rffi.charp2str(ll_source)
-        res = _pypy_execute_source(source)
-        before = rffi.aroundstate.before
-        if before: before()
-        return rffi.cast(rffi.INT, res)
+        return pypy_execute_source_ptr(ll_source, 0)
 
     @entrypoint('main', [rffi.CCHARP, lltype.Signed],
                 c_name='pypy_execute_source_ptr')
@@ -149,9 +143,7 @@
         after = rffi.aroundstate.after
         if after: after()
         source = rffi.charp2str(ll_source)
-        space.setitem(w_globals, space.wrap('c_argument'),
-                      space.wrap(ll_ptr))
-        res = _pypy_execute_source(source)
+        res = _pypy_execute_source(source, ll_ptr)
         before = rffi.aroundstate.before
         if before: before()
         return rffi.cast(rffi.INT, res)
@@ -176,15 +168,21 @@
         before = rffi.aroundstate.before
         if before: before()
 
-    w_globals = space.newdict()
-    space.setitem(w_globals, space.wrap('__builtins__'),
-                  space.builtin_modules['builtins'])
-
-    def _pypy_execute_source(source):
+    def _pypy_execute_source(source, c_argument):
         try:
-            compiler = space.createcompiler()
-            stmt = compiler.compile(source, 'c callback', 'exec', 0)
-            stmt.exec_code(space, w_globals, w_globals)
+            w_globals = space.newdict(module=True)
+            space.setitem(w_globals, space.wrap('__builtins__'),
+                          space.builtin_modules['__builtin__'])
+            space.setitem(w_globals, space.wrap('c_argument'),
+                          space.wrap(c_argument))
+            space.appexec([space.wrap(source), w_globals], """(src, glob):
+                import sys
+                stmt = compile(src, 'c callback', 'exec')
+                if not hasattr(sys, '_pypy_execute_source'):
+                    sys._pypy_execute_source = []
+                sys._pypy_execute_source.append(glob)
+                exec stmt in glob
+            """)
         except OperationError, e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -12,7 +12,7 @@
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
-    UserDelAction, CodeUniqueIds)
+    UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
@@ -398,7 +398,6 @@
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
-        self.code_unique_ids = CodeUniqueIds()
         self._code_of_sys_exc_info = None
 
         # can be overridden to a subclass
@@ -688,16 +687,6 @@
             assert ec is not None
             return ec
 
-    def register_code_callback(self, callback):
-        cui = self.code_unique_ids
-        cui.code_callback = callback
-
-    def register_code_object(self, pycode):
-        cui = self.code_unique_ids
-        if cui.code_callback is None:
-            return
-        cui.code_callback(self, pycode)
-
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -589,11 +589,3 @@
         # there is no list of length n: if n is large, then the GC
         # will run several times while walking the list, but it will
         # see lower and lower memory usage, with no lower bound of n.
-
-class CodeUniqueIds(object):
-    def __init__(self):
-        if sys.maxint == 2147483647:
-            self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
-        else:
-            self.code_unique_id = 0x7000000000000000
-        self.code_callback = None
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -105,7 +105,7 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
-        space.register_code_object(self)
+        self._init_ready()
 
     def _initialize(self):
         if self.co_cellvars:
@@ -148,14 +148,8 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
-        cui = self.space.code_unique_ids
-        self._unique_id = cui.code_unique_id
-        cui.code_unique_id += 4  # so we have two bits that we can mark stuff
-        # with
-
-    def _get_full_name(self):
-        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
-                                self.co_filename)
+    def _init_ready(self):
+        "This is a hook for the vmprof module, which overrides this method."
 
     def _cleanup_(self):
         if (self.magic == cpython_magic and
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -62,6 +62,7 @@
     }
 
     interpleveldefs = {
+        'attach_gdb'                : 'interp_magic.attach_gdb',
         'internal_repr'             : 'interp_magic.internal_repr',
         'bytebuffer'                : 'bytebuffer.bytebuffer',
         'identity_dict'             : 'interp_identitydict.W_IdentityDict',
@@ -98,8 +99,6 @@
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON"""
-        if not self.space.config.translating:
-            self.extra_interpdef('interp_pdb', 'interp_magic.interp_pdb')
         if self.space.config.objspace.std.withmethodcachecounter:
             self.extra_interpdef('method_cache_counter',
                                  'interp_magic.method_cache_counter')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -15,12 +15,10 @@
     return space.wrap('%r' % (w_object,))
 
 
-def interp_pdb(space):
-    """Run an interp-level pdb.
-    This is not available in translated versions of PyPy."""
-    assert not we_are_translated()
-    import pdb
-    pdb.set_trace()
+def attach_gdb(space):
+    """Run an interp-level gdb (or pdb when untranslated)"""
+    from rpython.rlib.debug import attach_gdb
+    attach_gdb()
 
 
 @unwrap_spec(name=str)
diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_file/readinto.py
@@ -0,0 +1,81 @@
+import sys, errno
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
+
+_WIN32 = sys.platform.startswith('win')
+UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
+
+os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
+                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
+                          rffi.SIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
+
+
+def direct_readinto(self, w_rwbuffer):
+    rwbuffer = self.space.writebuf_w(w_rwbuffer)
+    stream = self.getstream()
+    size = rwbuffer.getlength()
+    target_address = lltype.nullptr(rffi.CCHARP.TO)
+    fd = -1
+    target_pos = 0
+
+    if size > 64:
+        try:
+            target_address = rwbuffer.get_raw_address()
+        except ValueError:
+            pass
+        else:
+            fd = stream.try_to_find_file_descriptor()
+
+    if fd < 0 or not target_address:
+        # fall-back
+        MAX_PART = 1024 * 1024    # 1 MB
+        while size > MAX_PART:
+            data = self.direct_read(MAX_PART)
+            rwbuffer.setslice(target_pos, data)
+            target_pos += len(data)
+            size -= len(data)
+            if len(data) != MAX_PART:
+                break
+        else:
+            data = self.direct_read(size)
+            rwbuffer.setslice(target_pos, data)
+            target_pos += len(data)
+
+    else:
+        # optimized case: reading more than 64 bytes into a rwbuffer
+        # with a valid raw address
+        self.check_readable()
+
+        # first "read" the part that is already sitting in buffers, if any
+        initial_size = min(size, stream.count_buffered_bytes())
+        if initial_size > 0:
+            data = stream.read(initial_size)
+            rwbuffer.setslice(target_pos, data)
+            target_pos += len(data)
+            size -= len(data)
+
+        # then call os_read() to get the rest
+        if size > 0:
+            stream.flush()
+            while True:
+                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
+                if got > 0:
+                    target_pos += got
+                    size -= got
+                    if size <= 0:
+                        break
+                elif got == 0:
+                    break
+                else:
+                    err = rposix.get_saved_errno()
+                    if err == errno.EINTR:
+                        signal_checker(self.space)()
+                        continue
+                    if is_wouldblock_error(err) and target_pos > 0:
+                        break
+                    raise OSError(err, "read error")
+            keepalive_until_here(rwbuffer)
+
+    return self.space.wrap(target_pos)
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,8 +1,9 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib.rvmprof import VMProfPlatformUnsupported
 
 class Module(MixedModule):
     """
-    Write me :)
+    VMProf for PyPy: a statistical profiler
     """
     appleveldefs = {
     }
@@ -10,9 +11,16 @@
     interpleveldefs = {
         'enable': 'interp_vmprof.enable',
         'disable': 'interp_vmprof.disable',
+        'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError',
     }
 
-    def setup_after_space_initialization(self):
-        # force the __extend__ hacks to occur early
-        from pypy.module._vmprof.interp_vmprof import VMProf
-        self.vmprof = VMProf()
+
+# Force the __extend__ hacks and method replacements to occur
+# early.  Without this, for example, 'PyCode._init_ready' was
+# already found by the annotator to be the original empty
+# method, and the annotator doesn't notice that interp_vmprof.py
+# (loaded later) replaces this method.
+try:
+    import pypy.module._vmprof.interp_vmprof
+except VMProfPlatformUnsupported, e:
+    pass
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -1,252 +1,74 @@
-import py, os, sys
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, rgc
-from rpython.rlib.rarithmetic import ovfcheck_float_to_int
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib.rstring import StringBuilder
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.baseobjspace import W_Root
+from rpython.rlib import rvmprof
 
-ROOT = py.path.local(__file__).join('..')
-SRC = ROOT.join('src')
+# ____________________________________________________________
 
-# by default, we statically link vmprof.c into pypy; however, if you set
-# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
-# which is expected to be inside pypy/module/_vmprof/src: this is very useful
-# during development. Note that you have to manually build libvmprof by
-# running make inside the src dir
-DYNAMIC_VMPROF = False
 
-if sys.platform.startswith('linux'):
-    libs = ['dl']
-else:
-    libs = []
+_get_code = lambda frame, w_inputvalue, operr: frame.pycode
+_decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root)
+my_execute_frame = _decorator(PyFrame.execute_frame)
 
-eci_kwds = dict(
-    include_dirs = [SRC],
-    includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.vmprof.s')],
-    libraries = libs,
-    
-    post_include_bits=["""
-        int pypy_vmprof_init(void);
-    """],
-    
-    separate_module_sources=["""
-        int pypy_vmprof_init(void) {
-            return vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                NULL);
-        }
-    """],
-    )
-
-
-if DYNAMIC_VMPROF:
-    eci_kwds['libraries'] += ['vmprof']
-    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
-else:
-    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
-
-eci = ExternalCompilationInfo(**eci_kwds)
-
-check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
-    SRC.join('fake_pypy_api.c')]))
-
-platform.verify_eci(check_eci)
-
-pypy_execute_frame_trampoline = rffi.llexternal(
-    "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed],
-    llmemory.GCREF,
-    compilation_info=eci,
-    _nowrapper=True, sandboxsafe=True,
-    random_effects_on_gcobjs=True)
-
-pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], rffi.INT,
-                                   compilation_info=eci)
-vmprof_enable = rffi.llexternal("vmprof_enable",
-                                [rffi.INT, rffi.LONG, rffi.INT,
-                                 rffi.CCHARP, rffi.INT],
-                                rffi.INT, compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
-                                 compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-vmprof_get_error = rffi.llexternal("vmprof_get_error", [], rffi.CCHARP,
-                                   compilation_info=eci,
-                                   save_err=rffi.RFFI_SAVE_ERRNO)
-
-vmprof_register_virtual_function = rffi.llexternal(
-    "vmprof_register_virtual_function",
-    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    compilation_info=eci, _nowrapper=True)
-
-original_execute_frame = PyFrame.execute_frame.im_func
-original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
-original_execute_frame._dont_inline_ = True
 
 class __extend__(PyFrame):
-    def execute_frame(frame, w_inputvalue=None, operr=None):
-        # go through the asm trampoline ONLY if we are translated but not being JITted.
-        #
-        # If we are not translated, we obviously don't want to go through the
-        # trampoline because there is no C function it can call.
-        #
-        # If we are being JITted, we want to skip the trampoline, else the JIT
-        # cannot see throug it
-        if we_are_translated() and not jit.we_are_jitted():
-            # if we are translated, call the trampoline
-            gc_frame = cast_instance_to_gcref(frame)
-            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
-            gc_operr = cast_instance_to_gcref(operr)
-            unique_id = frame.pycode._unique_id
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
-                                                      gc_operr, unique_id)
-            return cast_base_ptr_to_instance(W_Root, gc_result)
-        else:
-            return original_execute_frame(frame, w_inputvalue, operr)
+    def execute_frame(self, w_inputvalue=None, operr=None):
+        # indirection for the optional arguments
+        return my_execute_frame(self, w_inputvalue, operr)
 
 
+def _safe(s):
+    if len(s) > 110:
+        s = s[:107] + '...'
+    return s.replace(':', ';')
 
-def write_long_to_string_builder(l, b):
-    if sys.maxint == 2147483647:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-    else:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
+def _get_full_name(pycode):
+    # careful, must not have extraneous ':' or be longer than 255 chars
+    return "py:%s:%d:%s" % (_safe(pycode.co_name), pycode.co_firstlineno,
+                            _safe(pycode.co_filename))
 
-def try_cast_to_pycode(gcref):
-    return rgc.try_cast_gcref_to_instance(PyCode, gcref)
+rvmprof.register_code_object_class(PyCode, _get_full_name)
 
-MAX_CODES = 1000
 
-class VMProf(object):
-    def __init__(self):
-        self.is_enabled = False
-        self.ever_enabled = False
-        self.fileno = -1
-        self.current_codes = []
+def _init_ready(pycode):
+    rvmprof.register_code(pycode, _get_full_name)
 
-    def enable(self, space, fileno, period_usec):
-        if self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof already enabled")
-        self.fileno = fileno
-        self.is_enabled = True
-        self.write_header(fileno, period_usec)
-        if not self.ever_enabled:
-            if we_are_translated():
-                res = pypy_vmprof_init()
-                if res:
-                    raise OperationError(
-                        space.w_IOError,
-                        space.wrap(rffi.charp2str(vmprof_get_error())))
-            self.ever_enabled = True
-        self.gather_all_code_objs(space)
-        space.register_code_callback(vmprof_register_code)
-        if we_are_translated():
-            # does not work untranslated
-            res = vmprof_enable(fileno, period_usec, 0,
-                                lltype.nullptr(rffi.CCHARP.TO), 0)
-        else:
-            res = 0
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.enable"))
+PyCode._init_ready = _init_ready
 
-    def gather_all_code_objs(self, space):
-        all_code_objs = rgc.do_get_objects(try_cast_to_pycode)
-        for code in all_code_objs:
-            self.register_code(space, code)
 
-    def write_header(self, fileno, period_usec):
-        assert period_usec > 0
-        b = StringBuilder()
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(3, b)
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(period_usec, b)
-        write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
-        os.write(fileno, b.build())
+# ____________________________________________________________
 
-    def register_code(self, space, code):
-        if self.fileno == -1:
-            raise OperationError(space.w_RuntimeError,
-                                 space.wrap("vmprof not running"))
-        self.current_codes.append(code)
-        if len(self.current_codes) >= MAX_CODES:
-            self._flush_codes(space)
 
-    def _flush_codes(self, space):
-        b = StringBuilder()
-        for code in self.current_codes:
-            name = code._get_full_name()
-            b.append('\x02')
-            write_long_to_string_builder(code._unique_id, b)
-            write_long_to_string_builder(len(name), b)
-            b.append(name)
-        os.write(self.fileno, b.build())
-        self.current_codes = []
+class Cache:
+    def __init__(self, space):
+        self.w_VMProfError = space.new_exception_class("_vmprof.VMProfError")
 
-    def disable(self, space):
-        if not self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof not enabled")
-        self.is_enabled = False
-        space.register_code_callback(None)
-        self._flush_codes(space)
-        self.fileno = -1
-        if we_are_translated():
-           # does not work untranslated
-            res = vmprof_disable()
-        else:
-            res = 0
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.disable"))
+def VMProfError(space, e):
+    w_VMProfError = space.fromcache(Cache).w_VMProfError
+    return OperationError(w_VMProfError, space.wrap(e.msg))
 
-def vmprof_register_code(space, code):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.register_code(space, code)
 
 @unwrap_spec(fileno=int, period=float)
-def enable(space, fileno, period=0.01):   # default 100 Hz
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    #
+def enable(space, fileno, period):
+    """Enable vmprof.  Writes go to the given 'fileno', a file descriptor
+    opened for writing.  *The file descriptor must remain open at least
+    until disable() is called.*
+
+    'interval' is a float representing the sampling interval, in seconds.
+    Must be smaller than 1.0
+    """
     try:
-        period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5)
-        if period_usec <= 0 or period_usec >= 1e6:
-            # we don't want seconds here at all
-            raise ValueError
-    except (ValueError, OverflowError):
-        raise OperationError(space.w_ValueError,
-                             space.wrap("'period' too large or non positive"))
-    #
-    mod_vmprof.vmprof.enable(space, fileno, period_usec)
+        rvmprof.enable(fileno, period)
+    except rvmprof.VMProfError, e:
+        raise VMProfError(space, e)
 
 def disable(space):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.disable(space)
-
+    """Disable vmprof.  Remember to close the file descriptor afterwards
+    if necessary.
+    """
+    try:
+        rvmprof.disable()
+    except rvmprof.VMProfError, e:
+        raise VMProfError(space, e)
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/config.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#define HAVE_SYS_UCONTEXT_H
-#if defined(__FreeBSD__) || defined(__APPLE__)
-#define PC_FROM_UCONTEXT uc_mcontext.mc_rip
-#else
-#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
-#endif
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ /dev/null
@@ -1,4 +0,0 @@
-
-void pypy_pyframe_execute_frame(void)
-{
-}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ /dev/null
@@ -1,80 +0,0 @@
-
-#ifdef PYPY_JIT_CODEMAP
-
-extern volatile int pypy_codemap_currently_invalid;
-
-void *pypy_find_codemap_at_addr(long addr, long *start_addr);
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr);
-long pypy_jit_stack_depth_at_loc(long loc);
-
-#endif
-
-
-void vmprof_set_tramp_range(void* start, void* end)
-{
-}
-
-int custom_sanity_check()
-{
-#ifdef PYPY_JIT_CODEMAP
-    return !pypy_codemap_currently_invalid;
-#else
-    return 1;
-#endif
-}
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
-    intptr_t ip_l = (intptr_t)ip;
-    return pypy_jit_stack_depth_at_loc(ip_l);
-#else
-    return -1;
-#endif
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-                                             void *ip, int max_depth)
-{
-#ifdef PYPY_JIT_CODEMAP
-    void *codemap;
-    long current_pos = 0;
-    intptr_t id;
-    long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    void *tmp;
-
-    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
-    if (codemap == NULL)
-        // not a jit code at all
-        return n;
-
-    // modify the last entry to point to start address and not the random one
-    // in the middle
-    result[n - 1] = (void*)start_addr;
-    result[n] = (void*)2;
-    n++;
-    start = n;
-    while (n < max_depth) {
-        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
-        if (id == -1)
-            // finish
-            break;
-        if (id == 0)
-            continue; // not main codemap
-        result[n++] = (void *)id;
-    }
-    k = 0;
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k - 1];
-        result[n - k - 1] = tmp;
-        k++;
-    }
-    if (n < max_depth) {
-        result[n++] = (void*)3;
-    }
-#endif
-    return n;
-}
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.h
+++ /dev/null
@@ -1,1 +0,0 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*, long);
diff --git a/pypy/module/_vmprof/src/trampoline.vmprof.s b/pypy/module/_vmprof/src/trampoline.vmprof.s
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.vmprof.s
+++ /dev/null
@@ -1,15 +0,0 @@
-// NOTE: you need to use TABs, not spaces!
-        
-	.text
-	.globl	pypy_execute_frame_trampoline
-	.type	pypy_execute_frame_trampoline, @function
-pypy_execute_frame_trampoline:
-	.cfi_startproc
-	pushq	%rcx
-	.cfi_def_cfa_offset 16
-	call pypy_pyframe_execute_frame at PLT
-	popq	%rcx
-	.cfi_def_cfa_offset 8
-	ret
-	.cfi_endproc
-	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.c
+++ /dev/null
@@ -1,463 +0,0 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *
- */
-
-
-#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-//#define UNW_LOCAL_ONLY
-//#include 
-
-#include "vmprof.h"
-#if defined(__FreeBSD__) || defined(__APPLE__)
-#define sighandler_t sig_t
-#endif
-
-#define _unused(x) ((void)x)
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 1024
-#define BUFFER_SIZE 8192
-
-
-static int profile_file = 0;
-static char profile_write_buffer[BUFFER_SIZE];
-static int profile_buffer_position = 0;
-void* vmprof_mainloop_func;
-char* vmprof_error = NULL;
-static ptrdiff_t mainloop_sp_offset;
-static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
-static long last_period_usec = 0;
-static int atfork_hook_installed = 0;
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-int (*unw_step)(unw_cursor_t*) = NULL;
-int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static void prof_word(long x) {
-	((long*)(profile_write_buffer + profile_buffer_position))[0] = x;
-	profile_buffer_position += sizeof(long);
-}
-
-static void prof_header(long period_usec) {
-    // XXX never used here?
-    prof_word(0);
-    prof_word(3);
-    prof_word(0);
-    prof_word(period_usec);
-    prof_word(0);
-    write(profile_file, profile_write_buffer, profile_buffer_position);
-    profile_buffer_position = 0;
-}
-
-static void prof_write_stacktrace(void** stack, int depth, int count) {
-    int i;
-	char marker = MARKER_STACKTRACE;
-
-	profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE;
-    prof_word(count);
-    prof_word(depth);
-    for(i=0; isp = bp;
-		bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-// The original code here has a comment, "stolen from pprof",
-// about a "__thread int recursive".  But general __thread
-// variables are not really supposed to be accessed from a
-// signal handler.  Moreover, we are using SIGPROF, which
-// should not be recursively called on the same thread.
-//static __thread int recursive;
-
-int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-    //if (recursive) {
-    //    return 0;
-    //}
-    if (!custom_sanity_check()) {
-        return 0;
-    }
-    //++recursive;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    _unused(ret);
-	int first_run = 1;
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* char funcname[4096]; */
-        /* unw_word_t offset; */
-        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
-        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (vmprof_mainloop_func && 
-            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
-            n > 0) {
-          // found main loop stack frame
-          void* sp;
-          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-          void *arg_addr = (char*)sp + mainloop_sp_offset;
-          void **arg_ptr = (void**)arg_addr;
-          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
-          //         sp, arg_addr, mainloop_sp_offset);
-		  if (mainloop_get_virtual_ip) {
-			  ip = mainloop_get_virtual_ip(*arg_ptr);
-		  } else {
-			  ip = *arg_ptr;
-		  }
-        }
-
-        result[n++] = ip;
-		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0) {
-            break;
-        }
-		first_run = 0;
-    }
-    //--recursive;
-    return n;
-}
-
-
-static int __attribute__((noinline)) frame_forcer(int rv) {
-    return rv;
-}
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
-    void* stack[MAX_STACK_DEPTH];
-    int saved_errno = errno;
-    stack[0] = GetPC((ucontext_t*)ucontext);
-    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
-    depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(stack, depth, 1);
-    errno = saved_errno;
-}
-
-/* *************************************************************
- * functions to enable/disable the profiler
- * *************************************************************
- */
-
-static int open_profile(int fd, long period_usec, int write_header, char *s,
-						int slen) {
-	if ((fd = dup(fd)) == -1) {
-		return -1;
-	}
-	profile_buffer_position = 0;
-    profile_file = fd;
-	if (write_header)
-		prof_header(period_usec);
-	if (s)
-		write(profile_file, s, slen);
-	return 0;
-}
-
-static int close_profile(void) {
-	// XXX all of this can happily fail
-    FILE* src;
-    char buf[BUFSIZ];
-    size_t size;
-	int marker = MARKER_TRAILER;
-	write(profile_file, &marker, 1);
-
-#ifdef __linux__
-    // copy /proc/PID/maps to the end of the profile file
-    sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");
-    if (!src) {
-        vmprof_error = "error opening proc maps";
-        return -1;
-    }
-    while ((size = fread(buf, 1, BUFSIZ, src))) {
-        write(profile_file, buf, size);
-    }
-    fclose(src);
-#else
-    // freebsd and mac
-    sprintf(buf, "procstat -v %d", getpid());
-    src = popen(buf, "r");
-    if (!src) {
-        vmprof_error = "error calling procstat";
-        return -1;
-    }
-    while ((size = fread(buf, 1, BUFSIZ, src))) {
-        write(profile_file, buf, size);
-    }
-    pclose(src);
-#endif
-    close(profile_file);
-	return 0;
-}
-
-
-static int install_sigprof_handler(void) {
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-		sigaction(SIGPROF, &sa, NULL) == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-static int remove_sigprof_handler(void) {
-    sighandler_t res = signal(SIGPROF, SIG_DFL);
-	if (res == SIG_ERR) {
-		return -1;
-	}
-	return 0;
-};
-
-static int install_sigprof_timer(long period_usec) {
-    static struct itimerval timer;
-    last_period_usec = period_usec;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = period_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (last_period_usec) {
-        remove_sigprof_timer();
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (last_period_usec) {
-        install_sigprof_timer(last_period_usec);
-    }
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-/* *************************************************************
- * public API
- * *************************************************************
- */
-
-int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip) {
-    void *libhandle;
-
-    mainloop_sp_offset = sp_offset;
-    mainloop_get_virtual_ip = get_virtual_ip;
-    vmprof_mainloop_func = func;
-    if (!unw_get_reg) {
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))){
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-    }
-    return 0;
-}
-
-char* vmprof_get_error()
-{
-    char* res;
-    res = vmprof_error;
-    vmprof_error = NULL;
-    return res;
-}
-
-int vmprof_enable(int fd, long period_usec, int write_header, char *s,
-				  int slen)
-{
-    assert(period_usec > 0);
-    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
-		return -1;
-	}
-    if (install_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (install_sigprof_timer(period_usec) == -1) {
-		return -1;
-	}
-    if (install_pthread_atfork_hooks() == -1) {
-        return -1;
-    }
-	return 0;
-}
-
-int vmprof_disable(void) {
-    if (remove_sigprof_timer() == -1) {
-		return -1;
-	}
-    last_period_usec = 0;
-    if (remove_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (close_profile() == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end) {
-	// XXX unused by pypy
-    // for now *end is simply ignored
-	char buf[1024];
-	int lgt = strlen(name) + 2 * sizeof(long) + 1;
-
-	if (lgt > 1024) {
-		lgt = 1024;
-	}
-	buf[0] = MARKER_VIRTUAL_IP;
-	((void **)(((void*)buf) + 1))[0] = start;
-	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
-	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	write(profile_file, buf, lgt);
-}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.h
+++ /dev/null
@@ -1,121 +0,0 @@
-#ifndef VMPROF_VMPROF_H_
-#define VMPROF_VMPROF_H_
-
-#include 
-#include 
-#include 
-
-// copied from libunwind.h
-
-typedef enum
-  {
-    UNW_X86_64_RAX,
-    UNW_X86_64_RDX,
-    UNW_X86_64_RCX,
-    UNW_X86_64_RBX,
-    UNW_X86_64_RSI,
-    UNW_X86_64_RDI,
-    UNW_X86_64_RBP,
-    UNW_X86_64_RSP,
-    UNW_X86_64_R8,
-    UNW_X86_64_R9,
-    UNW_X86_64_R10,
-    UNW_X86_64_R11,
-    UNW_X86_64_R12,
-    UNW_X86_64_R13,
-    UNW_X86_64_R14,
-    UNW_X86_64_R15,
-    UNW_X86_64_RIP,
-#ifdef CONFIG_MSABI_SUPPORT
-    UNW_X86_64_XMM0,
-    UNW_X86_64_XMM1,
-    UNW_X86_64_XMM2,
-    UNW_X86_64_XMM3,
-    UNW_X86_64_XMM4,
-    UNW_X86_64_XMM5,
-    UNW_X86_64_XMM6,
-    UNW_X86_64_XMM7,
-    UNW_X86_64_XMM8,
-    UNW_X86_64_XMM9,
-    UNW_X86_64_XMM10,
-    UNW_X86_64_XMM11,
-    UNW_X86_64_XMM12,
-    UNW_X86_64_XMM13,
-    UNW_X86_64_XMM14,
-    UNW_X86_64_XMM15,
-    UNW_TDEP_LAST_REG = UNW_X86_64_XMM15,
-#else
-    UNW_TDEP_LAST_REG = UNW_X86_64_RIP,
-#endif
-
-    /* XXX Add other regs here */
-
-    /* frame info (read-only) */
-    UNW_X86_64_CFA,
-
-    UNW_TDEP_IP = UNW_X86_64_RIP,
-    UNW_TDEP_SP = UNW_X86_64_RSP,
-    UNW_TDEP_BP = UNW_X86_64_RBP,
-    UNW_TDEP_EH = UNW_X86_64_RAX
-  }
-x86_64_regnum_t;
-
-typedef uint64_t unw_word_t;
-
-#define UNW_TDEP_CURSOR_LEN 127
-
-typedef struct unw_cursor
-  {
-    unw_word_t opaque[UNW_TDEP_CURSOR_LEN];
-  }
-unw_cursor_t;
-
-#define UNW_REG_IP UNW_X86_64_RIP
-#define UNW_REG_SP UNW_X86_64_RSP
-
-typedef ucontext_t unw_context_t;
-
-typedef struct unw_proc_info
-  {
-    unw_word_t start_ip;	/* first IP covered by this procedure */
-    unw_word_t end_ip;		/* first IP NOT covered by this procedure */
-    unw_word_t lsda;		/* address of lang.-spec. data area (if any) */
-    unw_word_t handler;		/* optional personality routine */
-    unw_word_t gp;		/* global-pointer value for this procedure */
-    unw_word_t flags;		/* misc. flags */
-
-    int format;			/* unwind-info format (arch-specific) */
-    int unwind_info_size;	/* size of the information (if applicable) */
-    void *unwind_info;		/* unwind-info (arch-specific) */
-  }
-unw_proc_info_t;
-
-// functions copied from libunwind using dlopen
-
-extern int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*);
-extern int (*unw_step)(unw_cursor_t*);
-extern int (*unw_init_local)(unw_cursor_t *, unw_context_t *);
-extern int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *);
-
-// end of copy
-
-extern char* vmprof_error;
-
-typedef void* (*vmprof_get_virtual_ip_t)(void*);
-char* vmprof_get_error();
-
-extern void* vmprof_mainloop_func;
-int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip);
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end);
-
-
-int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
-				  int vips_len);
-int vmprof_disable(void);
-
-// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
-void vmprof_set_tramp_range(void* start, void* end);
-
-#endif
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -1,14 +1,14 @@
 
-import tempfile
+from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
-        cls.tmpfile = tempfile.NamedTemporaryFile()
+        cls.tmpfile = udir.join('test__vmprof.1').open('wb')
         cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
         cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
-        cls.tmpfile2 = tempfile.NamedTemporaryFile()
+        cls.tmpfile2 = udir.join('test__vmprof.2').open('wb')
         cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
         cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
 
@@ -29,17 +29,21 @@
             while i < len(s):
                 if s[i] == 3:
                     break
-                if s[i] == 1:
-                    xxx
-                assert s[i] == 2
-                i += 1
-                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
-                count += 1
-                i += 2 * WORD + size
+                elif s[i] == 1:
+                    i += 1
+                    _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                    i += 2 * WORD + size * struct.calcsize("P")
+                elif s[i] == 2:
+                    i += 1
+                    _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                    count += 1
+                    i += 2 * WORD + size
+                else:
+                    raise AssertionError(ord(s[i]))
             return count
         
         import _vmprof
-        _vmprof.enable(self.tmpfileno)
+        _vmprof.enable(self.tmpfileno, 0.01)
         _vmprof.disable()
         s = open(self.tmpfilename, 'rb').read()
         no_of_codes = count(s)
@@ -53,7 +57,7 @@
             pass
         """, d)
 
-        _vmprof.enable(self.tmpfileno2)
+        _vmprof.enable(self.tmpfileno2, 0.01)
 
         exec_("""def foo2():
             pass
@@ -68,8 +72,9 @@
 
     def test_enable_ovf(self):
         import _vmprof
-        raises(ValueError, _vmprof.enable, 999, 0)
-        raises(ValueError, _vmprof.enable, 999, -2.5)
-        raises(ValueError, _vmprof.enable, 999, 1e300)
-        raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300)
-        raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300))
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, -2.5)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300 * 1e300)
+        NaN = (1e300*1e300) / (1e300*1e300)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, NaN)
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -5,7 +5,8 @@
 except ImportError:
     py.test.skip('cffi required')
 
-srcdir = py.path.local(__file__).join("..", "..", "src")
+from rpython.rlib import rvmprof
+srcdir = py.path.local(rvmprof.__file__).join("..", "src")
 
 ffi = cffi.FFI()
 ffi.cdef("""
@@ -17,6 +18,8 @@
 """)
 
 lib = ffi.verify("""
+#define PYPY_JIT_CODEMAP
+
 volatile int pypy_codemap_currently_invalid = 0;
 
 long buffer[] = {0, 0, 0, 0, 0};
@@ -39,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("get_custom_offset.c"))).read())
+""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/pypy/module/cpyext/TODO b/pypy/module/cpyext/TODO
deleted file mode 100644
--- a/pypy/module/cpyext/TODO
+++ /dev/null
@@ -1,26 +0,0 @@
- - Complete the PyTypeObject initialization code. (see XXX in the code)
-  - Implement further method callers.
-  - Copy the slots from the base.
-  - Those tasks are necessary to be able to call slots from C code correctly.
-  - Additionally, implement further slot wrappers. This is necessary to call
-    slots of PTOs defined in C.
- - Complete the Members support.
-
- - Use a WeakKeyDictionary to count how often a PyObject is allocated for
-   a given wrapped object and use this to assess whether optimizations are
-   useful
-
- - replace @cpython_api(external=False) by another explicit name: all
-   it does is a lltype function pointer, no C code involved.
-
- - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads).
-
- - refactor management of py_objects_r2w and py_objects_w2r, this can
-   probably be expressed in terms of _PyObject_GC_TRACK macros.
-
- - PyWeakref_GetObject() returns a borrowed reference, but this turns the
-   WeakRef into a strong reference!
-
- - sort out pypy's buffer protocol. PyPy's buffer right now don't support
-   raw memory (except array which supports it in a hackish way), which
-   should be fixed in order to make it nicely work with cpyext.
diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py
--- a/pypy/module/micronumpy/casting.py
+++ b/pypy/module/micronumpy/casting.py
@@ -145,23 +145,32 @@
     # equivalent to PyArray_CanCastTypeTo
     if origin == target:
         return True
-    if origin.is_record() or target.is_record():
-        return can_cast_record(space, origin, target, casting)
+    if casting == 'unsafe':
+        return True
+    elif casting == 'no':
+        return origin.eq(space, target)
+    if origin.num == target.num:
+        if origin.is_record():
+            return (target.is_record() and
+                    can_cast_record(space, origin, target, casting))
+        else:
+            if casting == 'equiv':
+                return origin.elsize == target.elsize
+            elif casting == 'safe':
+                return origin.elsize <= target.elsize
+            else:
+                return True
 
-    if casting == 'no':
-        return origin.eq(space, target)
-    elif casting == 'equiv':
-        return origin.num == target.num and origin.elsize == target.elsize
-    elif casting == 'unsafe':
-        return True
     elif casting == 'same_kind':
         if can_cast_to(origin, target):
             return True
         if origin.kind in kind_ordering and target.kind in kind_ordering:
             return kind_ordering[origin.kind] <= kind_ordering[target.kind]
         return False
-    else:  # 'safe'
+    elif casting == 'safe':
         return can_cast_to(origin, target)
+    else:  # 'equiv'
+        return origin.num == target.num and origin.elsize == target.elsize
 
 def can_cast_record(space, origin, target, casting):
     if origin is target:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -52,11 +52,16 @@
 
     @jit.unroll_safe
     def setslice(self, space, arr):
-        if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
-            raise oefmt(space.w_ValueError,
-                "could not broadcast input array from shape "
-                "(%s) into shape ()",
-                ','.join([str(x) for x in arr.get_shape()]))
+        if len(arr.get_shape()) >  len(self.get_shape()):
+            # record arrays get one extra dimension
+            if not self.dtype.is_record() or \
+                    len(arr.get_shape()) > len(self.get_shape()) + 1:
+                raise oefmt(space.w_ValueError,
+                    "could not broadcast input array from shape "
+                    "(%s) into shape (%s)",
+                    ','.join([str(x) for x in arr.get_shape()]),
+                    ','.join([str(x) for x in self.get_shape()]),
+                    )
         shape = shape_agreement(space, self.get_shape(), arr)
         impl = arr.implementation
         if impl.storage == self.storage:
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -589,7 +589,8 @@
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
-        if self.fields is None:  # if builtin dtype
+        if self.fields is None and not isinstance(self.itemtype, types.VoidType):  
+            # if builtin dtype (but not w_voiddtype)
             return space.w_None
 
         version = space.int_w(space.getitem(w_data, space.wrap(0)))
@@ -1030,6 +1031,11 @@
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
+        if space.isinstance_w(w_dtype0, space.w_type) and \
+           space.isinstance_w(w_dtype1, space.w_list):
+            #obscure api - (subclass, spec). Ignore the subclass
+            return make_new_dtype(space, w_subtype, w_dtype1, alignment, 
+                        copy=copy, w_shape=w_shape, w_metadata=w_metadata)
         subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -204,17 +204,16 @@
         self.array.setitem(state.offset, elem)
 
 
-def AxisIter(array, shape, axis, cumulative):
+def AxisIter(array, shape, axis):
     strides = array.get_strides()
     backstrides = array.get_backstrides()
-    if not cumulative:
-        if len(shape) == len(strides):
-            # keepdims = True
-            strides = strides[:axis] + [0] + strides[axis + 1:]
-            backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
-        else:
-            strides = strides[:axis] + [0] + strides[axis:]
-            backstrides = backstrides[:axis] + [0] + backstrides[axis:]
+    if len(shape) == len(strides):
+        # keepdims = True
+        strides = strides[:axis] + [0] + strides[axis + 1:]
+        backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
+    else:
+        strides = strides[:axis] + [0] + strides[axis:]
+        backstrides = backstrides[:axis] + [0] + backstrides[axis:]
     return ArrayIter(array, support.product(shape), shape, strides, backstrides)
 
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -9,7 +9,7 @@
 from pypy.module.micronumpy import support, constants as NPY
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \
-    AllButAxisIter
+    AllButAxisIter, ArrayIter
 from pypy.interpreter.argument import Arguments
 
 
@@ -190,23 +190,64 @@
         source_state = source_iter.next(source_state)
     return target
 
-reduce_driver = jit.JitDriver(name='numpy_reduce',
-                              greens = ['shapelen', 'func', 'done_func',
-                                        'calc_dtype'],
-                              reds = 'auto')
 
-def compute_reduce(space, obj, calc_dtype, func, done_func, identity):
-    obj_iter, obj_state = obj.create_iter()
+def split_iter(arr, axis_flags):
+    """Prepare 2 iterators for nested iteration over `arr`.
+
+    Arguments:
+        arr: instance of BaseConcreteArray
+        axis_flags: list of bools, one for each dimension of `arr`.The inner
+        iterator operates over the dimensions for which the flag is True
+    """
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    backstrides = arr.get_backstrides()
+    shapelen = len(shape)
+    assert len(axis_flags) == shapelen
+    inner_shape = [-1] * shapelen
+    inner_strides = [-1] * shapelen
+    inner_backstrides = [-1] * shapelen
+    outer_shape = [-1] * shapelen
+    outer_strides = [-1] * shapelen
+    outer_backstrides = [-1] * shapelen
+    for i in range(len(shape)):
+        if axis_flags[i]:
+            inner_shape[i] = shape[i]
+            inner_strides[i] = strides[i]
+            inner_backstrides[i] = backstrides[i]
+            outer_shape[i] = 1
+            outer_strides[i] = 0
+            outer_backstrides[i] = 0
+        else:
+            outer_shape[i] = shape[i]
+            outer_strides[i] = strides[i]
+            outer_backstrides[i] = backstrides[i]
+            inner_shape[i] = 1
+            inner_strides[i] = 0
+            inner_backstrides[i] = 0
+    inner_iter = ArrayIter(arr, support.product(inner_shape),
+                           inner_shape, inner_strides, inner_backstrides)
+    outer_iter = ArrayIter(arr, support.product(outer_shape),
+                           outer_shape, outer_strides, outer_backstrides)
+    return inner_iter, outer_iter
+
+
+reduce_flat_driver = jit.JitDriver(
+    name='numpy_reduce_flat',
+    greens = ['shapelen', 'func', 'done_func', 'calc_dtype'], reds = 'auto')
+
+def reduce_flat(space, func, w_arr, calc_dtype, done_func, identity):
+    obj_iter, obj_state = w_arr.create_iter()
     if identity is None:
         cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         obj_state = obj_iter.next(obj_state)
     else:
         cur_value = identity.convert_to(space, calc_dtype)
-    shapelen = len(obj.get_shape())
+    shapelen = len(w_arr.get_shape())
     while not obj_iter.done(obj_state):
-        reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                      done_func=done_func,
-                                      calc_dtype=calc_dtype)
+        reduce_flat_driver.jit_merge_point(
+            shapelen=shapelen, func=func,
+            done_func=done_func, calc_dtype=calc_dtype)
         rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         if done_func is not None and done_func(calc_dtype, rval):
             return rval
@@ -214,33 +255,105 @@
         obj_state = obj_iter.next(obj_state)
     return cur_value
 
-reduce_cum_driver = jit.JitDriver(
-    name='numpy_reduce_cum_driver',
+
+reduce_driver = jit.JitDriver(
+    name='numpy_reduce',
+    greens=['shapelen', 'func', 'dtype'], reds='auto')
+
+def reduce(space, func, w_arr, axis_flags, dtype, out, identity):
+    out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
+    shape = w_arr.get_shape()
+    shapelen = len(shape)
+    inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags)
+    assert outer_iter.size == out_iter.size
+
+    if identity is not None:
+        identity = identity.convert_to(space, dtype)
+    outer_state = outer_iter.reset()
+    while not outer_iter.done(outer_state):
+        inner_state = inner_iter.reset()
+        inner_state.offset = outer_state.offset
+        if identity is not None:
+            w_val = identity
+        else:
+            w_val = inner_iter.getitem(inner_state).convert_to(space, dtype)
+            inner_state = inner_iter.next(inner_state)
+        while not inner_iter.done(inner_state):
+            reduce_driver.jit_merge_point(
+                shapelen=shapelen, func=func, dtype=dtype)
+            w_item = inner_iter.getitem(inner_state).convert_to(space, dtype)
+            w_val = func(dtype, w_item, w_val)
+            inner_state = inner_iter.next(inner_state)
+        out_iter.setitem(out_state, w_val)
+        out_state = out_iter.next(out_state)
+        outer_state = outer_iter.next(outer_state)
+    return out
+
+accumulate_flat_driver = jit.JitDriver(
+    name='numpy_accumulate_flat',
     greens=['shapelen', 'func', 'dtype', 'out_dtype'],
     reds='auto')
 
-def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
-    obj_iter, obj_state = obj.create_iter()
-    out_iter, out_state = out.create_iter()
+def accumulate_flat(space, func, w_arr, calc_dtype, w_out, identity):
+    arr_iter, arr_state = w_arr.create_iter()
+    out_iter, out_state = w_out.create_iter()
     out_iter.track_index = False
     if identity is None:
-        cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
+        cur_value = arr_iter.getitem(arr_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, cur_value)
         out_state = out_iter.next(out_state)
-        obj_state = obj_iter.next(obj_state)
+        arr_state = arr_iter.next(arr_state)
     else:
         cur_value = identity.convert_to(space, calc_dtype)
-    shapelen = len(obj.get_shape())
-    out_dtype = out.get_dtype()
-    while not obj_iter.done(obj_state):
-        reduce_cum_driver.jit_merge_point(
-            shapelen=shapelen, func=func,
-            dtype=calc_dtype, out_dtype=out_dtype)
-        rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
-        cur_value = func(calc_dtype, cur_value, rval)
+    shapelen = len(w_arr.get_shape())
+    out_dtype = w_out.get_dtype()
+    while not arr_iter.done(arr_state):
+        accumulate_flat_driver.jit_merge_point(
+            shapelen=shapelen, func=func, dtype=calc_dtype,
+            out_dtype=out_dtype)
+        w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype)
+        cur_value = func(calc_dtype, cur_value, w_item)
         out_iter.setitem(out_state, out_dtype.coerce(space, cur_value))
         out_state = out_iter.next(out_state)
-        obj_state = obj_iter.next(obj_state)
+        arr_state = arr_iter.next(arr_state)
+
+accumulate_driver = jit.JitDriver(
+    name='numpy_accumulate',
+    greens=['shapelen', 'func', 'calc_dtype'], reds='auto')
+
+
+def accumulate(space, func, w_arr, axis, calc_dtype, w_out, identity):
+    out_iter, out_state = w_out.create_iter()
+    arr_shape = w_arr.get_shape()
+    temp_shape = arr_shape[:axis] + arr_shape[axis + 1:]
+    temp = W_NDimArray.from_shape(space, temp_shape, calc_dtype, w_instance=w_arr)
+    temp_iter = AxisIter(temp.implementation, w_arr.get_shape(), axis)
+    temp_state = temp_iter.reset()
+    arr_iter, arr_state = w_arr.create_iter()
+    arr_iter.track_index = False
+    if identity is not None:
+        identity = identity.convert_to(space, calc_dtype)
+    shapelen = len(arr_shape)
+    while not out_iter.done(out_state):
+        accumulate_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                          calc_dtype=calc_dtype)
+        w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype)
+        arr_state = arr_iter.next(arr_state)
+
+        out_indices = out_iter.indices(out_state)
+        if out_indices[axis] == 0:
+            if identity is not None:
+                w_item = func(calc_dtype, identity, w_item)
+        else:
+            cur_value = temp_iter.getitem(temp_state)
+            w_item = func(calc_dtype, cur_value, w_item)
+
+        out_iter.setitem(out_state, w_item)
+        out_state = out_iter.next(out_state)
+        temp_iter.setitem(temp_state, w_item)
+        temp_state = temp_iter.next(temp_state)
+    return w_out
 
 def fill(arr, box):
     arr_iter, arr_state = arr.create_iter()
@@ -298,64 +411,56 @@
             state = x_state
     return out
 
-axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce',
-                                   greens=['shapelen', 'func', 'dtype'],
-                                   reds='auto')
-
-def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative,
-                   temp):
-    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative)
-    out_state = out_iter.reset()
-    if cumulative:
-        temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
-        temp_state = temp_iter.reset()
-    else:
-        temp_iter = out_iter  # hack
-        temp_state = out_state
-    arr_iter, arr_state = arr.create_iter()
-    arr_iter.track_index = False
-    if identity is not None:
-        identity = identity.convert_to(space, dtype)
-    shapelen = len(shape)
-    while not out_iter.done(out_state):
-        axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                           dtype=dtype)
-        w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
-        arr_state = arr_iter.next(arr_state)
-
-        out_indices = out_iter.indices(out_state)
-        if out_indices[axis] == 0:
-            if identity is not None:
-                w_val = func(dtype, identity, w_val)
-        else:
-            cur = temp_iter.getitem(temp_state)
-            w_val = func(dtype, cur, w_val)
-
-        out_iter.setitem(out_state, w_val)
-        out_state = out_iter.next(out_state)
-        if cumulative:
-            temp_iter.setitem(temp_state, w_val)
-            temp_state = temp_iter.next(temp_state)
-        else:
-            temp_state = out_state
-    return out
-
 
 def _new_argmin_argmax(op_name):
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
                                reds = 'auto')
+    arg_flat_driver = jit.JitDriver(name='numpy_flat_' + op_name,
+                                    greens = ['shapelen', 'dtype'],
+                                    reds = 'auto')
 
-    def argmin_argmax(arr):
+    def argmin_argmax(space, w_arr, w_out, axis):
+        from pypy.module.micronumpy.descriptor import get_dtype_cache
+        dtype = w_arr.get_dtype()
+        shapelen = len(w_arr.get_shape())
+        axis_flags = [False] * shapelen
+        axis_flags[axis] = True
+        inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags)
+        outer_state = outer_iter.reset()
+        out_iter, out_state = w_out.create_iter()
+        while not outer_iter.done(outer_state):
+            inner_state = inner_iter.reset()
+            inner_state.offset = outer_state.offset
+            cur_best = inner_iter.getitem(inner_state)
+            inner_state = inner_iter.next(inner_state)
+            result = 0
+            idx = 1
+            while not inner_iter.done(inner_state):
+                arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+                w_val = inner_iter.getitem(inner_state)
+                new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
+                if dtype.itemtype.ne(new_best, cur_best):
+                    result = idx
+                    cur_best = new_best
+                inner_state = inner_iter.next(inner_state)
+                idx += 1
+            result = get_dtype_cache(space).w_longdtype.box(result)
+            out_iter.setitem(out_state, result)
+            out_state = out_iter.next(out_state)
+            outer_state = outer_iter.next(outer_state)
+        return w_out
+
+    def argmin_argmax_flat(w_arr):
         result = 0
         idx = 1
-        dtype = arr.get_dtype()
-        iter, state = arr.create_iter()
+        dtype = w_arr.get_dtype()
+        iter, state = w_arr.create_iter()
         cur_best = iter.getitem(state)
         state = iter.next(state)
-        shapelen = len(arr.get_shape())
+        shapelen = len(w_arr.get_shape())
         while not iter.done(state):
-            arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+            arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
             w_val = iter.getitem(state)
             new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
             if dtype.itemtype.ne(new_best, cur_best):
@@ -364,9 +469,10 @@
             state = iter.next(state)
             idx += 1
         return result
-    return argmin_argmax
-argmin = _new_argmin_argmax('min')
-argmax = _new_argmin_argmax('max')
+
+    return argmin_argmax, argmin_argmax_flat
+argmin, argmin_flat = _new_argmin_argmax('min')
+argmax, argmax_flat = _new_argmin_argmax('max')
 
 dot_driver = jit.JitDriver(name = 'numpy_dot',
                            greens = ['dtype'],
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -23,6 +23,8 @@
     get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
     is_c_contiguous, is_f_contiguous, calc_strides, new_view)
 from pypy.module.micronumpy.casting import can_cast_array
+from pypy.module.micronumpy.descriptor import get_dtype_cache
+
 
 
 def _match_dot_shapes(space, left, right):
@@ -245,6 +247,20 @@
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
+        # TODO: multiarray/mapping.c calls a subclass's __getitem__ here, which
+        # is a big performance hit but necessary for the matrix class. The original
+        # C code is like:
+        #/*
+        #* WARNING: There is a huge special case here. If this is not a
+        #*          base class array, we have to get the view through its
+        #*          very own index machinery.
+        #*          Many subclasses should probably call __setitem__
+        #*          with a base class ndarray view to avoid this.
+        #*/
+        #else if (!(index_type & (HAS_FANCY | HAS_SCALAR_ARRAY))
+        #        && !PyArray_CheckExact(self)) {
+        #view = (PyArrayObject *)PyObject_GetItem((PyObject *)self, ind);
+
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
             self.setitem_filter(space, w_idx, convert_to_array(space, w_value))

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:41 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150817121541.94B1A1C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79011:58654aa3cd80
Date: 2015-08-17 02:41 +0200
http://bitbucket.org/pypy/pypy/changeset/58654aa3cd80/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 5812 lines

diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("= 2.6.1,
+   you get a new globals dictionary for every call (but then, all globals
+   dictionaries are all kept alive forever, in ``sys._pypy_execute_source``).
 
 .. function:: int pypy_execute_source_ptr(char* source, void* ptr);
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -55,3 +55,15 @@
 .. branch: nditer-revisited
 
 Implement nditer 'buffered' flag and fix some edge cases
+
+.. branch: ufunc-reduce
+
+Allow multiple axes in ufunc.reduce()
+
+.. branch: fix-tinylang-goals
+
+Update tinylang goals to match current rpython
+
+.. branch: vmprof-review
+
+Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -135,13 +135,7 @@
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
-        after = rffi.aroundstate.after
-        if after: after()
-        source = rffi.charp2str(ll_source)
-        res = _pypy_execute_source(source)
-        before = rffi.aroundstate.before
-        if before: before()
-        return rffi.cast(rffi.INT, res)
+        return pypy_execute_source_ptr(ll_source, 0)
 
     @entrypoint('main', [rffi.CCHARP, lltype.Signed],
                 c_name='pypy_execute_source_ptr')
@@ -149,9 +143,7 @@
         after = rffi.aroundstate.after
         if after: after()
         source = rffi.charp2str(ll_source)
-        space.setitem(w_globals, space.wrap('c_argument'),
-                      space.wrap(ll_ptr))
-        res = _pypy_execute_source(source)
+        res = _pypy_execute_source(source, ll_ptr)
         before = rffi.aroundstate.before
         if before: before()
         return rffi.cast(rffi.INT, res)
@@ -176,15 +168,21 @@
         before = rffi.aroundstate.before
         if before: before()
 
-    w_globals = space.newdict()
-    space.setitem(w_globals, space.wrap('__builtins__'),
-                  space.builtin_modules['builtins'])
-
-    def _pypy_execute_source(source):
+    def _pypy_execute_source(source, c_argument):
         try:
-            compiler = space.createcompiler()
-            stmt = compiler.compile(source, 'c callback', 'exec', 0)
-            stmt.exec_code(space, w_globals, w_globals)
+            w_globals = space.newdict(module=True)
+            space.setitem(w_globals, space.wrap('__builtins__'),
+                          space.builtin_modules['__builtin__'])
+            space.setitem(w_globals, space.wrap('c_argument'),
+                          space.wrap(c_argument))
+            space.appexec([space.wrap(source), w_globals], """(src, glob):
+                import sys
+                stmt = compile(src, 'c callback', 'exec')
+                if not hasattr(sys, '_pypy_execute_source'):
+                    sys._pypy_execute_source = []
+                sys._pypy_execute_source.append(glob)
+                exec stmt in glob
+            """)
         except OperationError, e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space).encode('utf-8'))
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -12,7 +12,7 @@
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
-    UserDelAction, CodeUniqueIds)
+    UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
@@ -398,7 +398,6 @@
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
-        self.code_unique_ids = CodeUniqueIds()
         self._code_of_sys_exc_info = None
 
         # can be overridden to a subclass
@@ -690,16 +689,6 @@
             assert ec is not None
             return ec
 
-    def register_code_callback(self, callback):
-        cui = self.code_unique_ids
-        cui.code_callback = callback
-
-    def register_code_object(self, pycode):
-        cui = self.code_unique_ids
-        if cui.code_callback is None:
-            return
-        cui.code_callback(self, pycode)
-
     def _freeze_(self):
         return True
 
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -589,11 +589,3 @@
         # there is no list of length n: if n is large, then the GC
         # will run several times while walking the list, but it will
         # see lower and lower memory usage, with no lower bound of n.
-
-class CodeUniqueIds(object):
-    def __init__(self):
-        if sys.maxint == 2147483647:
-            self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
-        else:
-            self.code_unique_id = 0x7000000000000000
-        self.code_callback = None
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -109,7 +109,7 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
-        space.register_code_object(self)
+        self._init_ready()
 
     def _initialize(self):
         if self.co_cellvars:
@@ -152,14 +152,8 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
-        cui = self.space.code_unique_ids
-        self._unique_id = cui.code_unique_id
-        cui.code_unique_id += 4  # so we have two bits that we can mark stuff
-        # with
-
-    def _get_full_name(self):
-        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
-                                self.co_filename)
+    def _init_ready(self):
+        "This is a hook for the vmprof module, which overrides this method."
 
     def _cleanup_(self):
         if (self.magic == cpython_magic and
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -62,6 +62,7 @@
     }
 
     interpleveldefs = {
+        'attach_gdb'                : 'interp_magic.attach_gdb',
         'internal_repr'             : 'interp_magic.internal_repr',
         'bytebuffer'                : 'bytebuffer.bytebuffer',
         'identity_dict'             : 'interp_identitydict.W_IdentityDict',
@@ -98,8 +99,6 @@
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON"""
-        if not self.space.config.translating:
-            self.extra_interpdef('interp_pdb', 'interp_magic.interp_pdb')
         if self.space.config.objspace.std.withmethodcachecounter:
             self.extra_interpdef('method_cache_counter',
                                  'interp_magic.method_cache_counter')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -15,12 +15,10 @@
     return space.wrap('%r' % (w_object,))
 
 
-def interp_pdb(space):
-    """Run an interp-level pdb.
-    This is not available in translated versions of PyPy."""
-    assert not we_are_translated()
-    import pdb
-    pdb.set_trace()
+def attach_gdb(space):
+    """Run an interp-level gdb (or pdb when untranslated)"""
+    from rpython.rlib.debug import attach_gdb
+    attach_gdb()
 
 
 @unwrap_spec(name=str)
diff --git a/pypy/module/_file/readinto.py b/pypy/module/_file/readinto.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_file/readinto.py
@@ -0,0 +1,81 @@
+import sys, errno
+from rpython.rlib import rposix
+from rpython.rlib.objectmodel import keepalive_until_here
+from rpython.rtyper.lltypesystem import lltype, rffi
+from pypy.module._file.interp_file import is_wouldblock_error, signal_checker
+
+_WIN32 = sys.platform.startswith('win')
+UNDERSCORE_ON_WIN32 = '_' if _WIN32 else ''
+
+os_read = rffi.llexternal(UNDERSCORE_ON_WIN32 + 'read',
+                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
+                          rffi.SIZE_T, save_err=rffi.RFFI_SAVE_ERRNO)
+
+
+def direct_readinto(self, w_rwbuffer):
+    rwbuffer = self.space.writebuf_w(w_rwbuffer)
+    stream = self.getstream()
+    size = rwbuffer.getlength()
+    target_address = lltype.nullptr(rffi.CCHARP.TO)
+    fd = -1
+    target_pos = 0
+
+    if size > 64:
+        try:
+            target_address = rwbuffer.get_raw_address()
+        except ValueError:
+            pass
+        else:
+            fd = stream.try_to_find_file_descriptor()
+
+    if fd < 0 or not target_address:
+        # fall-back
+        MAX_PART = 1024 * 1024    # 1 MB
+        while size > MAX_PART:
+            data = self.direct_read(MAX_PART)
+            rwbuffer.setslice(target_pos, data)
+            target_pos += len(data)
+            size -= len(data)
+            if len(data) != MAX_PART:
+                break
+        else:
+            data = self.direct_read(size)
+            rwbuffer.setslice(target_pos, data)
+            target_pos += len(data)
+
+    else:
+        # optimized case: reading more than 64 bytes into a rwbuffer
+        # with a valid raw address
+        self.check_readable()
+
+        # first "read" the part that is already sitting in buffers, if any
+        initial_size = min(size, stream.count_buffered_bytes())
+        if initial_size > 0:
+            data = stream.read(initial_size)
+            rwbuffer.setslice(target_pos, data)
+            target_pos += len(data)
+            size -= len(data)
+
+        # then call os_read() to get the rest
+        if size > 0:
+            stream.flush()
+            while True:
+                got = os_read(fd, rffi.ptradd(target_address, target_pos), size)
+                if got > 0:
+                    target_pos += got
+                    size -= got
+                    if size <= 0:
+                        break
+                elif got == 0:
+                    break
+                else:
+                    err = rposix.get_saved_errno()
+                    if err == errno.EINTR:
+                        signal_checker(self.space)()
+                        continue
+                    if is_wouldblock_error(err) and target_pos > 0:
+                        break
+                    raise OSError(err, "read error")
+            keepalive_until_here(rwbuffer)
+
+    return self.space.wrap(target_pos)
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,8 +1,9 @@
 from pypy.interpreter.mixedmodule import MixedModule
+from rpython.rlib.rvmprof import VMProfPlatformUnsupported
 
 class Module(MixedModule):
     """
-    Write me :)
+    VMProf for PyPy: a statistical profiler
     """
     appleveldefs = {
     }
@@ -10,9 +11,16 @@
     interpleveldefs = {
         'enable': 'interp_vmprof.enable',
         'disable': 'interp_vmprof.disable',
+        'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError',
     }
 
-    def setup_after_space_initialization(self):
-        # force the __extend__ hacks to occur early
-        from pypy.module._vmprof.interp_vmprof import VMProf
-        self.vmprof = VMProf()
+
+# Force the __extend__ hacks and method replacements to occur
+# early.  Without this, for example, 'PyCode._init_ready' was
+# already found by the annotator to be the original empty
+# method, and the annotator doesn't notice that interp_vmprof.py
+# (loaded later) replaces this method.
+try:
+    import pypy.module._vmprof.interp_vmprof
+except VMProfPlatformUnsupported, e:
+    pass
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -1,252 +1,74 @@
-import py, os, sys
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
-from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref, cast_base_ptr_to_instance
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib import jit, rposix, rgc
-from rpython.rlib.rarithmetic import ovfcheck_float_to_int
-from rpython.rtyper.tool import rffi_platform as platform
-from rpython.rlib.rstring import StringBuilder
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import oefmt, wrap_oserror, OperationError
+from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.baseobjspace import W_Root
+from rpython.rlib import rvmprof
 
-ROOT = py.path.local(__file__).join('..')
-SRC = ROOT.join('src')
+# ____________________________________________________________
 
-# by default, we statically link vmprof.c into pypy; however, if you set
-# DYNAMIC_VMPROF to True, it will be dynamically linked to the libvmprof.so
-# which is expected to be inside pypy/module/_vmprof/src: this is very useful
-# during development. Note that you have to manually build libvmprof by
-# running make inside the src dir
-DYNAMIC_VMPROF = False
 
-if sys.platform.startswith('linux'):
-    libs = ['dl']
-else:
-    libs = []
+_get_code = lambda frame, w_inputvalue, operr: frame.pycode
+_decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root)
+my_execute_frame = _decorator(PyFrame.execute_frame)
 
-eci_kwds = dict(
-    include_dirs = [SRC],
-    includes = ['vmprof.h', 'trampoline.h'],
-    separate_module_files = [SRC.join('trampoline.vmprof.s')],
-    libraries = libs,
-    
-    post_include_bits=["""
-        int pypy_vmprof_init(void);
-    """],
-    
-    separate_module_sources=["""
-        int pypy_vmprof_init(void) {
-            return vmprof_set_mainloop(pypy_execute_frame_trampoline, 0,
-                                NULL);
-        }
-    """],
-    )
-
-
-if DYNAMIC_VMPROF:
-    eci_kwds['libraries'] += ['vmprof']
-    eci_kwds['link_extra'] = ['-Wl,-rpath,%s' % SRC, '-L%s' % SRC]
-else:
-    eci_kwds['separate_module_files'] += [SRC.join('vmprof.c')]
-
-eci = ExternalCompilationInfo(**eci_kwds)
-
-check_eci = eci.merge(ExternalCompilationInfo(separate_module_files=[
-    SRC.join('fake_pypy_api.c')]))
-
-platform.verify_eci(check_eci)
-
-pypy_execute_frame_trampoline = rffi.llexternal(
-    "pypy_execute_frame_trampoline",
-    [llmemory.GCREF, llmemory.GCREF, llmemory.GCREF, lltype.Signed],
-    llmemory.GCREF,
-    compilation_info=eci,
-    _nowrapper=True, sandboxsafe=True,
-    random_effects_on_gcobjs=True)
-
-pypy_vmprof_init = rffi.llexternal("pypy_vmprof_init", [], rffi.INT,
-                                   compilation_info=eci)
-vmprof_enable = rffi.llexternal("vmprof_enable",
-                                [rffi.INT, rffi.LONG, rffi.INT,
-                                 rffi.CCHARP, rffi.INT],
-                                rffi.INT, compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
-                                 compilation_info=eci,
-                                save_err=rffi.RFFI_SAVE_ERRNO)
-vmprof_get_error = rffi.llexternal("vmprof_get_error", [], rffi.CCHARP,
-                                   compilation_info=eci,
-                                   save_err=rffi.RFFI_SAVE_ERRNO)
-
-vmprof_register_virtual_function = rffi.llexternal(
-    "vmprof_register_virtual_function",
-    [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    compilation_info=eci, _nowrapper=True)
-
-original_execute_frame = PyFrame.execute_frame.im_func
-original_execute_frame.c_name = 'pypy_pyframe_execute_frame'
-original_execute_frame._dont_inline_ = True
 
 class __extend__(PyFrame):
-    def execute_frame(frame, w_inputvalue=None, operr=None):
-        # go through the asm trampoline ONLY if we are translated but not being JITted.
-        #
-        # If we are not translated, we obviously don't want to go through the
-        # trampoline because there is no C function it can call.
-        #
-        # If we are being JITted, we want to skip the trampoline, else the JIT
-        # cannot see throug it
-        if we_are_translated() and not jit.we_are_jitted():
-            # if we are translated, call the trampoline
-            gc_frame = cast_instance_to_gcref(frame)
-            gc_inputvalue = cast_instance_to_gcref(w_inputvalue)
-            gc_operr = cast_instance_to_gcref(operr)
-            unique_id = frame.pycode._unique_id
-            gc_result = pypy_execute_frame_trampoline(gc_frame, gc_inputvalue,
-                                                      gc_operr, unique_id)
-            return cast_base_ptr_to_instance(W_Root, gc_result)
-        else:
-            return original_execute_frame(frame, w_inputvalue, operr)
+    def execute_frame(self, w_inputvalue=None, operr=None):
+        # indirection for the optional arguments
+        return my_execute_frame(self, w_inputvalue, operr)
 
 
+def _safe(s):
+    if len(s) > 110:
+        s = s[:107] + '...'
+    return s.replace(':', ';')
 
-def write_long_to_string_builder(l, b):
-    if sys.maxint == 2147483647:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-    else:
-        b.append(chr(l & 0xff))
-        b.append(chr((l >> 8) & 0xff))
-        b.append(chr((l >> 16) & 0xff))
-        b.append(chr((l >> 24) & 0xff))
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
+def _get_full_name(pycode):
+    # careful, must not have extraneous ':' or be longer than 255 chars
+    return "py:%s:%d:%s" % (_safe(pycode.co_name), pycode.co_firstlineno,
+                            _safe(pycode.co_filename))
 
-def try_cast_to_pycode(gcref):
-    return rgc.try_cast_gcref_to_instance(PyCode, gcref)
+rvmprof.register_code_object_class(PyCode, _get_full_name)
 
-MAX_CODES = 1000
 
-class VMProf(object):
-    def __init__(self):
-        self.is_enabled = False
-        self.ever_enabled = False
-        self.fileno = -1
-        self.current_codes = []
+def _init_ready(pycode):
+    rvmprof.register_code(pycode, _get_full_name)
 
-    def enable(self, space, fileno, period_usec):
-        if self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof already enabled")
-        self.fileno = fileno
-        self.is_enabled = True
-        self.write_header(fileno, period_usec)
-        if not self.ever_enabled:
-            if we_are_translated():
-                res = pypy_vmprof_init()
-                if res:
-                    raise OperationError(
-                        space.w_IOError,
-                        space.wrap(rffi.charp2str(vmprof_get_error())))
-            self.ever_enabled = True
-        self.gather_all_code_objs(space)
-        space.register_code_callback(vmprof_register_code)
-        if we_are_translated():
-            # does not work untranslated
-            res = vmprof_enable(fileno, period_usec, 0,
-                                lltype.nullptr(rffi.CCHARP.TO), 0)
-        else:
-            res = 0
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.enable"))
+PyCode._init_ready = _init_ready
 
-    def gather_all_code_objs(self, space):
-        all_code_objs = rgc.do_get_objects(try_cast_to_pycode)
-        for code in all_code_objs:
-            self.register_code(space, code)
 
-    def write_header(self, fileno, period_usec):
-        assert period_usec > 0
-        b = StringBuilder()
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(3, b)
-        write_long_to_string_builder(0, b)
-        write_long_to_string_builder(period_usec, b)
-        write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
-        os.write(fileno, b.build())
+# ____________________________________________________________
 
-    def register_code(self, space, code):
-        if self.fileno == -1:
-            raise OperationError(space.w_RuntimeError,
-                                 space.wrap("vmprof not running"))
-        self.current_codes.append(code)
-        if len(self.current_codes) >= MAX_CODES:
-            self._flush_codes(space)
 
-    def _flush_codes(self, space):
-        b = StringBuilder()
-        for code in self.current_codes:
-            name = code._get_full_name()
-            b.append('\x02')
-            write_long_to_string_builder(code._unique_id, b)
-            write_long_to_string_builder(len(name), b)
-            b.append(name)
-        os.write(self.fileno, b.build())
-        self.current_codes = []
+class Cache:
+    def __init__(self, space):
+        self.w_VMProfError = space.new_exception_class("_vmprof.VMProfError")
 
-    def disable(self, space):
-        if not self.is_enabled:
-            raise oefmt(space.w_ValueError, "_vmprof not enabled")
-        self.is_enabled = False
-        space.register_code_callback(None)
-        self._flush_codes(space)
-        self.fileno = -1
-        if we_are_translated():
-           # does not work untranslated
-            res = vmprof_disable()
-        else:
-            res = 0
-        if res == -1:
-            raise wrap_oserror(space, OSError(rposix.get_saved_errno(),
-                                              "_vmprof.disable"))
+def VMProfError(space, e):
+    w_VMProfError = space.fromcache(Cache).w_VMProfError
+    return OperationError(w_VMProfError, space.wrap(e.msg))
 
-def vmprof_register_code(space, code):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.register_code(space, code)
 
 @unwrap_spec(fileno=int, period=float)
-def enable(space, fileno, period=0.01):   # default 100 Hz
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    #
+def enable(space, fileno, period):
+    """Enable vmprof.  Writes go to the given 'fileno', a file descriptor
+    opened for writing.  *The file descriptor must remain open at least
+    until disable() is called.*
+
+    'interval' is a float representing the sampling interval, in seconds.
+    Must be smaller than 1.0
+    """
     try:
-        period_usec = ovfcheck_float_to_int(period * 1000000.0 + 0.5)
-        if period_usec <= 0 or period_usec >= 1e6:
-            # we don't want seconds here at all
-            raise ValueError
-    except (ValueError, OverflowError):
-        raise OperationError(space.w_ValueError,
-                             space.wrap("'period' too large or non positive"))
-    #
-    mod_vmprof.vmprof.enable(space, fileno, period_usec)
+        rvmprof.enable(fileno, period)
+    except rvmprof.VMProfError, e:
+        raise VMProfError(space, e)
 
 def disable(space):
-    from pypy.module._vmprof import Module
-    mod_vmprof = space.getbuiltinmodule('_vmprof')
-    assert isinstance(mod_vmprof, Module)
-    mod_vmprof.vmprof.disable(space)
-
+    """Disable vmprof.  Remember to close the file descriptor afterwards
+    if necessary.
+    """
+    try:
+        rvmprof.disable()
+    except rvmprof.VMProfError, e:
+        raise VMProfError(space, e)
diff --git a/pypy/module/_vmprof/src/config.h b/pypy/module/_vmprof/src/config.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/config.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#define HAVE_SYS_UCONTEXT_H
-#if defined(__FreeBSD__) || defined(__APPLE__)
-#define PC_FROM_UCONTEXT uc_mcontext.mc_rip
-#else
-#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
-#endif
diff --git a/pypy/module/_vmprof/src/fake_pypy_api.c b/pypy/module/_vmprof/src/fake_pypy_api.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/fake_pypy_api.c
+++ /dev/null
@@ -1,4 +0,0 @@
-
-void pypy_pyframe_execute_frame(void)
-{
-}
diff --git a/pypy/module/_vmprof/src/get_custom_offset.c b/pypy/module/_vmprof/src/get_custom_offset.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/get_custom_offset.c
+++ /dev/null
@@ -1,80 +0,0 @@
-
-#ifdef PYPY_JIT_CODEMAP
-
-extern volatile int pypy_codemap_currently_invalid;
-
-void *pypy_find_codemap_at_addr(long addr, long *start_addr);
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr);
-long pypy_jit_stack_depth_at_loc(long loc);
-
-#endif
-
-
-void vmprof_set_tramp_range(void* start, void* end)
-{
-}
-
-int custom_sanity_check()
-{
-#ifdef PYPY_JIT_CODEMAP
-    return !pypy_codemap_currently_invalid;
-#else
-    return 1;
-#endif
-}
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
-    intptr_t ip_l = (intptr_t)ip;
-    return pypy_jit_stack_depth_at_loc(ip_l);
-#else
-    return -1;
-#endif
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-                                             void *ip, int max_depth)
-{
-#ifdef PYPY_JIT_CODEMAP
-    void *codemap;
-    long current_pos = 0;
-    intptr_t id;
-    long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    void *tmp;
-
-    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
-    if (codemap == NULL)
-        // not a jit code at all
-        return n;
-
-    // modify the last entry to point to start address and not the random one
-    // in the middle
-    result[n - 1] = (void*)start_addr;
-    result[n] = (void*)2;
-    n++;
-    start = n;
-    while (n < max_depth) {
-        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
-        if (id == -1)
-            // finish
-            break;
-        if (id == 0)
-            continue; // not main codemap
-        result[n++] = (void *)id;
-    }
-    k = 0;
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k - 1];
-        result[n - k - 1] = tmp;
-        k++;
-    }
-    if (n < max_depth) {
-        result[n++] = (void*)3;
-    }
-#endif
-    return n;
-}
diff --git a/pypy/module/_vmprof/src/trampoline.h b/pypy/module/_vmprof/src/trampoline.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.h
+++ /dev/null
@@ -1,1 +0,0 @@
-void* pypy_execute_frame_trampoline(void*, void*, void*, long);
diff --git a/pypy/module/_vmprof/src/trampoline.vmprof.s b/pypy/module/_vmprof/src/trampoline.vmprof.s
deleted file mode 100644
--- a/pypy/module/_vmprof/src/trampoline.vmprof.s
+++ /dev/null
@@ -1,15 +0,0 @@
-// NOTE: you need to use TABs, not spaces!
-        
-	.text
-	.globl	pypy_execute_frame_trampoline
-	.type	pypy_execute_frame_trampoline, @function
-pypy_execute_frame_trampoline:
-	.cfi_startproc
-	pushq	%rcx
-	.cfi_def_cfa_offset 16
-	call pypy_pyframe_execute_frame at PLT
-	popq	%rcx
-	.cfi_def_cfa_offset 8
-	ret
-	.cfi_endproc
-	.size	pypy_execute_frame_trampoline, .-pypy_execute_frame_trampoline
diff --git a/pypy/module/_vmprof/src/vmprof.c b/pypy/module/_vmprof/src/vmprof.c
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.c
+++ /dev/null
@@ -1,463 +0,0 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *
- */
-
-
-#include "getpc.h"      // should be first to get the _GNU_SOURCE dfn
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-
-//#define UNW_LOCAL_ONLY
-//#include 
-
-#include "vmprof.h"
-#if defined(__FreeBSD__) || defined(__APPLE__)
-#define sighandler_t sig_t
-#endif
-
-#define _unused(x) ((void)x)
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH 1024
-#define BUFFER_SIZE 8192
-
-
-static int profile_file = 0;
-static char profile_write_buffer[BUFFER_SIZE];
-static int profile_buffer_position = 0;
-void* vmprof_mainloop_func;
-char* vmprof_error = NULL;
-static ptrdiff_t mainloop_sp_offset;
-static vmprof_get_virtual_ip_t mainloop_get_virtual_ip;
-static long last_period_usec = 0;
-static int atfork_hook_installed = 0;
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-int (*unw_step)(unw_cursor_t*) = NULL;
-int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static void prof_word(long x) {
-	((long*)(profile_write_buffer + profile_buffer_position))[0] = x;
-	profile_buffer_position += sizeof(long);
-}
-
-static void prof_header(long period_usec) {
-    // XXX never used here?
-    prof_word(0);
-    prof_word(3);
-    prof_word(0);
-    prof_word(period_usec);
-    prof_word(0);
-    write(profile_file, profile_write_buffer, profile_buffer_position);
-    profile_buffer_position = 0;
-}
-
-static void prof_write_stacktrace(void** stack, int depth, int count) {
-    int i;
-	char marker = MARKER_STACKTRACE;
-
-	profile_write_buffer[profile_buffer_position++] = MARKER_STACKTRACE;
-    prof_word(count);
-    prof_word(depth);
-    for(i=0; isp = bp;
-		bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-// The original code here has a comment, "stolen from pprof",
-// about a "__thread int recursive".  But general __thread
-// variables are not really supposed to be accessed from a
-// signal handler.  Moreover, we are using SIGPROF, which
-// should not be recursively called on the same thread.
-//static __thread int recursive;
-
-int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) {
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-    //if (recursive) {
-    //    return 0;
-    //}
-    if (!custom_sanity_check()) {
-        return 0;
-    }
-    //++recursive;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    _unused(ret);
-	int first_run = 1;
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* char funcname[4096]; */
-        /* unw_word_t offset; */
-        /* unw_get_proc_name(&cursor, funcname, 4096, &offset); */
-        /* printf("%s+%#lx <%p>\n", funcname, offset, ip); */
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (vmprof_mainloop_func && 
-            (void*)pip.start_ip == (void*)vmprof_mainloop_func &&
-            n > 0) {
-          // found main loop stack frame
-          void* sp;
-          unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-          void *arg_addr = (char*)sp + mainloop_sp_offset;
-          void **arg_ptr = (void**)arg_addr;
-          // fprintf(stderr, "stacktrace mainloop: rsp %p   &f2 %p   offset %ld\n", 
-          //         sp, arg_addr, mainloop_sp_offset);
-		  if (mainloop_get_virtual_ip) {
-			  ip = mainloop_get_virtual_ip(*arg_ptr);
-		  } else {
-			  ip = *arg_ptr;
-		  }
-        }
-
-        result[n++] = ip;
-		n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0) {
-            break;
-        }
-		first_run = 0;
-    }
-    //--recursive;
-    return n;
-}
-
-
-static int __attribute__((noinline)) frame_forcer(int rv) {
-    return rv;
-}
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) {
-    void* stack[MAX_STACK_DEPTH];
-    int saved_errno = errno;
-    stack[0] = GetPC((ucontext_t*)ucontext);
-    int depth = frame_forcer(get_stack_trace(stack+1, MAX_STACK_DEPTH-1, ucontext));
-    depth++;  // To account for pc value in stack[0];
-    prof_write_stacktrace(stack, depth, 1);
-    errno = saved_errno;
-}
-
-/* *************************************************************
- * functions to enable/disable the profiler
- * *************************************************************
- */
-
-static int open_profile(int fd, long period_usec, int write_header, char *s,
-						int slen) {
-	if ((fd = dup(fd)) == -1) {
-		return -1;
-	}
-	profile_buffer_position = 0;
-    profile_file = fd;
-	if (write_header)
-		prof_header(period_usec);
-	if (s)
-		write(profile_file, s, slen);
-	return 0;
-}
-
-static int close_profile(void) {
-	// XXX all of this can happily fail
-    FILE* src;
-    char buf[BUFSIZ];
-    size_t size;
-	int marker = MARKER_TRAILER;
-	write(profile_file, &marker, 1);
-
-#ifdef __linux__
-    // copy /proc/PID/maps to the end of the profile file
-    sprintf(buf, "/proc/%d/maps", getpid());
-    src = fopen(buf, "r");
-    if (!src) {
-        vmprof_error = "error opening proc maps";
-        return -1;
-    }
-    while ((size = fread(buf, 1, BUFSIZ, src))) {
-        write(profile_file, buf, size);
-    }
-    fclose(src);
-#else
-    // freebsd and mac
-    sprintf(buf, "procstat -v %d", getpid());
-    src = popen(buf, "r");
-    if (!src) {
-        vmprof_error = "error calling procstat";
-        return -1;
-    }
-    while ((size = fread(buf, 1, BUFSIZ, src))) {
-        write(profile_file, buf, size);
-    }
-    pclose(src);
-#endif
-    close(profile_file);
-	return 0;
-}
-
-
-static int install_sigprof_handler(void) {
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-		sigaction(SIGPROF, &sa, NULL) == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-static int remove_sigprof_handler(void) {
-    sighandler_t res = signal(SIGPROF, SIG_DFL);
-	if (res == SIG_ERR) {
-		return -1;
-	}
-	return 0;
-};
-
-static int install_sigprof_timer(long period_usec) {
-    static struct itimerval timer;
-    last_period_usec = period_usec;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = period_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
-		return -1;
-    }
-	return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (last_period_usec) {
-        remove_sigprof_timer();
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (last_period_usec) {
-        install_sigprof_timer(last_period_usec);
-    }
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-/* *************************************************************
- * public API
- * *************************************************************
- */
-
-int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip) {
-    void *libhandle;
-
-    mainloop_sp_offset = sp_offset;
-    mainloop_get_virtual_ip = get_virtual_ip;
-    vmprof_mainloop_func = func;
-    if (!unw_get_reg) {
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))){
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) {
-            vmprof_error = dlerror();
-            return -1;
-        }
-    }
-    return 0;
-}
-
-char* vmprof_get_error()
-{
-    char* res;
-    res = vmprof_error;
-    vmprof_error = NULL;
-    return res;
-}
-
-int vmprof_enable(int fd, long period_usec, int write_header, char *s,
-				  int slen)
-{
-    assert(period_usec > 0);
-    if (open_profile(fd, period_usec, write_header, s, slen) == -1) {
-		return -1;
-	}
-    if (install_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (install_sigprof_timer(period_usec) == -1) {
-		return -1;
-	}
-    if (install_pthread_atfork_hooks() == -1) {
-        return -1;
-    }
-	return 0;
-}
-
-int vmprof_disable(void) {
-    if (remove_sigprof_timer() == -1) {
-		return -1;
-	}
-    last_period_usec = 0;
-    if (remove_sigprof_handler() == -1) {
-		return -1;
-	}
-    if (close_profile() == -1) {
-		return -1;
-	}
-	return 0;
-}
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end) {
-	// XXX unused by pypy
-    // for now *end is simply ignored
-	char buf[1024];
-	int lgt = strlen(name) + 2 * sizeof(long) + 1;
-
-	if (lgt > 1024) {
-		lgt = 1024;
-	}
-	buf[0] = MARKER_VIRTUAL_IP;
-	((void **)(((void*)buf) + 1))[0] = start;
-	((long *)(((void*)buf) + 1 + sizeof(long)))[0] = lgt - 2 * sizeof(long) - 1;
-	strncpy(buf + 2 * sizeof(long) + 1, name, 1024 - 2 * sizeof(long) - 1);
-	write(profile_file, buf, lgt);
-}
diff --git a/pypy/module/_vmprof/src/vmprof.h b/pypy/module/_vmprof/src/vmprof.h
deleted file mode 100644
--- a/pypy/module/_vmprof/src/vmprof.h
+++ /dev/null
@@ -1,121 +0,0 @@
-#ifndef VMPROF_VMPROF_H_
-#define VMPROF_VMPROF_H_
-
-#include 
-#include 
-#include 
-
-// copied from libunwind.h
-
-typedef enum
-  {
-    UNW_X86_64_RAX,
-    UNW_X86_64_RDX,
-    UNW_X86_64_RCX,
-    UNW_X86_64_RBX,
-    UNW_X86_64_RSI,
-    UNW_X86_64_RDI,
-    UNW_X86_64_RBP,
-    UNW_X86_64_RSP,
-    UNW_X86_64_R8,
-    UNW_X86_64_R9,
-    UNW_X86_64_R10,
-    UNW_X86_64_R11,
-    UNW_X86_64_R12,
-    UNW_X86_64_R13,
-    UNW_X86_64_R14,
-    UNW_X86_64_R15,
-    UNW_X86_64_RIP,
-#ifdef CONFIG_MSABI_SUPPORT
-    UNW_X86_64_XMM0,
-    UNW_X86_64_XMM1,
-    UNW_X86_64_XMM2,
-    UNW_X86_64_XMM3,
-    UNW_X86_64_XMM4,
-    UNW_X86_64_XMM5,
-    UNW_X86_64_XMM6,
-    UNW_X86_64_XMM7,
-    UNW_X86_64_XMM8,
-    UNW_X86_64_XMM9,
-    UNW_X86_64_XMM10,
-    UNW_X86_64_XMM11,
-    UNW_X86_64_XMM12,
-    UNW_X86_64_XMM13,
-    UNW_X86_64_XMM14,
-    UNW_X86_64_XMM15,
-    UNW_TDEP_LAST_REG = UNW_X86_64_XMM15,
-#else
-    UNW_TDEP_LAST_REG = UNW_X86_64_RIP,
-#endif
-
-    /* XXX Add other regs here */
-
-    /* frame info (read-only) */
-    UNW_X86_64_CFA,
-
-    UNW_TDEP_IP = UNW_X86_64_RIP,
-    UNW_TDEP_SP = UNW_X86_64_RSP,
-    UNW_TDEP_BP = UNW_X86_64_RBP,
-    UNW_TDEP_EH = UNW_X86_64_RAX
-  }
-x86_64_regnum_t;
-
-typedef uint64_t unw_word_t;
-
-#define UNW_TDEP_CURSOR_LEN 127
-
-typedef struct unw_cursor
-  {
-    unw_word_t opaque[UNW_TDEP_CURSOR_LEN];
-  }
-unw_cursor_t;
-
-#define UNW_REG_IP UNW_X86_64_RIP
-#define UNW_REG_SP UNW_X86_64_RSP
-
-typedef ucontext_t unw_context_t;
-
-typedef struct unw_proc_info
-  {
-    unw_word_t start_ip;	/* first IP covered by this procedure */
-    unw_word_t end_ip;		/* first IP NOT covered by this procedure */
-    unw_word_t lsda;		/* address of lang.-spec. data area (if any) */
-    unw_word_t handler;		/* optional personality routine */
-    unw_word_t gp;		/* global-pointer value for this procedure */
-    unw_word_t flags;		/* misc. flags */
-
-    int format;			/* unwind-info format (arch-specific) */
-    int unwind_info_size;	/* size of the information (if applicable) */
-    void *unwind_info;		/* unwind-info (arch-specific) */
-  }
-unw_proc_info_t;
-
-// functions copied from libunwind using dlopen
-
-extern int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*);
-extern int (*unw_step)(unw_cursor_t*);
-extern int (*unw_init_local)(unw_cursor_t *, unw_context_t *);
-extern int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *);
-
-// end of copy
-
-extern char* vmprof_error;
-
-typedef void* (*vmprof_get_virtual_ip_t)(void*);
-char* vmprof_get_error();
-
-extern void* vmprof_mainloop_func;
-int vmprof_set_mainloop(void* func, ptrdiff_t sp_offset, 
-                         vmprof_get_virtual_ip_t get_virtual_ip);
-
-void vmprof_register_virtual_function(const char* name, void* start, void* end);
-
-
-int vmprof_enable(int fd, long period_usec, int write_header, char* vips,
-				  int vips_len);
-int vmprof_disable(void);
-
-// XXX: this should be part of _vmprof (the CPython extension), not vmprof (the library)
-void vmprof_set_tramp_range(void* start, void* end);
-
-#endif
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -1,14 +1,14 @@
 
-import tempfile
+from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 
 class AppTestVMProf(object):
     def setup_class(cls):
         cls.space = gettestobjspace(usemodules=['_vmprof', 'struct'])
-        cls.tmpfile = tempfile.NamedTemporaryFile()
+        cls.tmpfile = udir.join('test__vmprof.1').open('wb')
         cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno())
         cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name)
-        cls.tmpfile2 = tempfile.NamedTemporaryFile()
+        cls.tmpfile2 = udir.join('test__vmprof.2').open('wb')
         cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno())
         cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name)
 
@@ -29,17 +29,21 @@
             while i < len(s):
                 if s[i] == 3:
                     break
-                if s[i] == 1:
-                    xxx
-                assert s[i] == 2
-                i += 1
-                _, size = struct.unpack("ll", s[i:i + 2 * WORD])
-                count += 1
-                i += 2 * WORD + size
+                elif s[i] == 1:
+                    i += 1
+                    _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                    i += 2 * WORD + size * struct.calcsize("P")
+                elif s[i] == 2:
+                    i += 1
+                    _, size = struct.unpack("ll", s[i:i + 2 * WORD])
+                    count += 1
+                    i += 2 * WORD + size
+                else:
+                    raise AssertionError(ord(s[i]))
             return count
         
         import _vmprof
-        _vmprof.enable(self.tmpfileno)
+        _vmprof.enable(self.tmpfileno, 0.01)
         _vmprof.disable()
         s = open(self.tmpfilename, 'rb').read()
         no_of_codes = count(s)
@@ -53,7 +57,7 @@
             pass
         """, d)
 
-        _vmprof.enable(self.tmpfileno2)
+        _vmprof.enable(self.tmpfileno2, 0.01)
 
         exec_("""def foo2():
             pass
@@ -68,8 +72,9 @@
 
     def test_enable_ovf(self):
         import _vmprof
-        raises(ValueError, _vmprof.enable, 999, 0)
-        raises(ValueError, _vmprof.enable, 999, -2.5)
-        raises(ValueError, _vmprof.enable, 999, 1e300)
-        raises(ValueError, _vmprof.enable, 999, 1e300 * 1e300)
-        raises(ValueError, _vmprof.enable, 999, (1e300*1e300) / (1e300*1e300))
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, -2.5)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, 1e300 * 1e300)
+        NaN = (1e300*1e300) / (1e300*1e300)
+        raises(_vmprof.VMProfError, _vmprof.enable, 999, NaN)
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -5,7 +5,8 @@
 except ImportError:
     py.test.skip('cffi required')
 
-srcdir = py.path.local(__file__).join("..", "..", "src")
+from rpython.rlib import rvmprof
+srcdir = py.path.local(rvmprof.__file__).join("..", "src")
 
 ffi = cffi.FFI()
 ffi.cdef("""
@@ -17,6 +18,8 @@
 """)
 
 lib = ffi.verify("""
+#define PYPY_JIT_CODEMAP
+
 volatile int pypy_codemap_currently_invalid = 0;
 
 long buffer[] = {0, 0, 0, 0, 0};
@@ -39,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("get_custom_offset.c"))).read())
+""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/pypy/module/cpyext/TODO b/pypy/module/cpyext/TODO
deleted file mode 100644
--- a/pypy/module/cpyext/TODO
+++ /dev/null
@@ -1,26 +0,0 @@
- - Complete the PyTypeObject initialization code. (see XXX in the code)
-  - Implement further method callers.
-  - Copy the slots from the base.
-  - Those tasks are necessary to be able to call slots from C code correctly.
-  - Additionally, implement further slot wrappers. This is necessary to call
-    slots of PTOs defined in C.
- - Complete the Members support.
-
- - Use a WeakKeyDictionary to count how often a PyObject is allocated for
-   a given wrapped object and use this to assess whether optimizations are
-   useful
-
- - replace @cpython_api(external=False) by another explicit name: all
-   it does is a lltype function pointer, no C code involved.
-
- - Fix GIL handling (e.g. after releasing the GIL, GC operations might occur in savethreads).
-
- - refactor management of py_objects_r2w and py_objects_w2r, this can
-   probably be expressed in terms of _PyObject_GC_TRACK macros.
-
- - PyWeakref_GetObject() returns a borrowed reference, but this turns the
-   WeakRef into a strong reference!
-
- - sort out pypy's buffer protocol. PyPy's buffer right now don't support
-   raw memory (except array which supports it in a hackish way), which
-   should be fixed in order to make it nicely work with cpyext.
diff --git a/pypy/module/micronumpy/casting.py b/pypy/module/micronumpy/casting.py
--- a/pypy/module/micronumpy/casting.py
+++ b/pypy/module/micronumpy/casting.py
@@ -145,23 +145,32 @@
     # equivalent to PyArray_CanCastTypeTo
     if origin == target:
         return True
-    if origin.is_record() or target.is_record():
-        return can_cast_record(space, origin, target, casting)
+    if casting == 'unsafe':
+        return True
+    elif casting == 'no':
+        return origin.eq(space, target)
+    if origin.num == target.num:
+        if origin.is_record():
+            return (target.is_record() and
+                    can_cast_record(space, origin, target, casting))
+        else:
+            if casting == 'equiv':
+                return origin.elsize == target.elsize
+            elif casting == 'safe':
+                return origin.elsize <= target.elsize
+            else:
+                return True
 
-    if casting == 'no':
-        return origin.eq(space, target)
-    elif casting == 'equiv':
-        return origin.num == target.num and origin.elsize == target.elsize
-    elif casting == 'unsafe':
-        return True
     elif casting == 'same_kind':
         if can_cast_to(origin, target):
             return True
         if origin.kind in kind_ordering and target.kind in kind_ordering:
             return kind_ordering[origin.kind] <= kind_ordering[target.kind]
         return False
-    else:  # 'safe'
+    elif casting == 'safe':
         return can_cast_to(origin, target)
+    else:  # 'equiv'
+        return origin.num == target.num and origin.elsize == target.elsize
 
 def can_cast_record(space, origin, target, casting):
     if origin is target:
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -52,11 +52,16 @@
 
     @jit.unroll_safe
     def setslice(self, space, arr):
-        if len(arr.get_shape()) > 0 and len(self.get_shape()) == 0:
-            raise oefmt(space.w_ValueError,
-                "could not broadcast input array from shape "
-                "(%s) into shape ()",
-                ','.join([str(x) for x in arr.get_shape()]))
+        if len(arr.get_shape()) >  len(self.get_shape()):
+            # record arrays get one extra dimension
+            if not self.dtype.is_record() or \
+                    len(arr.get_shape()) > len(self.get_shape()) + 1:
+                raise oefmt(space.w_ValueError,
+                    "could not broadcast input array from shape "
+                    "(%s) into shape (%s)",
+                    ','.join([str(x) for x in arr.get_shape()]),
+                    ','.join([str(x) for x in self.get_shape()]),
+                    )
         shape = shape_agreement(space, self.get_shape(), arr)
         impl = arr.implementation
         if impl.storage == self.storage:
diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py
--- a/pypy/module/micronumpy/descriptor.py
+++ b/pypy/module/micronumpy/descriptor.py
@@ -589,7 +589,8 @@
         return space.newtuple([w_class, builder_args, data])
 
     def descr_setstate(self, space, w_data):
-        if self.fields is None:  # if builtin dtype
+        if self.fields is None and not isinstance(self.itemtype, types.VoidType):  
+            # if builtin dtype (but not w_voiddtype)
             return space.w_None
 
         version = space.int_w(space.getitem(w_data, space.wrap(0)))
@@ -1030,6 +1031,11 @@
     elif space.isinstance_w(w_dtype, space.w_tuple):
         w_dtype0 = space.getitem(w_dtype, space.wrap(0))
         w_dtype1 = space.getitem(w_dtype, space.wrap(1))
+        if space.isinstance_w(w_dtype0, space.w_type) and \
+           space.isinstance_w(w_dtype1, space.w_list):
+            #obscure api - (subclass, spec). Ignore the subclass
+            return make_new_dtype(space, w_subtype, w_dtype1, alignment, 
+                        copy=copy, w_shape=w_shape, w_metadata=w_metadata)
         subdtype = make_new_dtype(space, w_subtype, w_dtype0, alignment, copy)
         assert isinstance(subdtype, W_Dtype)
         if subdtype.elsize == 0:
diff --git a/pypy/module/micronumpy/iterators.py b/pypy/module/micronumpy/iterators.py
--- a/pypy/module/micronumpy/iterators.py
+++ b/pypy/module/micronumpy/iterators.py
@@ -204,17 +204,16 @@
         self.array.setitem(state.offset, elem)
 
 
-def AxisIter(array, shape, axis, cumulative):
+def AxisIter(array, shape, axis):
     strides = array.get_strides()
     backstrides = array.get_backstrides()
-    if not cumulative:
-        if len(shape) == len(strides):
-            # keepdims = True
-            strides = strides[:axis] + [0] + strides[axis + 1:]
-            backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
-        else:
-            strides = strides[:axis] + [0] + strides[axis:]
-            backstrides = backstrides[:axis] + [0] + backstrides[axis:]
+    if len(shape) == len(strides):
+        # keepdims = True
+        strides = strides[:axis] + [0] + strides[axis + 1:]
+        backstrides = backstrides[:axis] + [0] + backstrides[axis + 1:]
+    else:
+        strides = strides[:axis] + [0] + strides[axis:]
+        backstrides = backstrides[:axis] + [0] + backstrides[axis:]
     return ArrayIter(array, support.product(shape), shape, strides, backstrides)
 
 
diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py
--- a/pypy/module/micronumpy/loop.py
+++ b/pypy/module/micronumpy/loop.py
@@ -9,7 +9,7 @@
 from pypy.module.micronumpy import support, constants as NPY
 from pypy.module.micronumpy.base import W_NDimArray, convert_to_array
 from pypy.module.micronumpy.iterators import PureShapeIter, AxisIter, \
-    AllButAxisIter
+    AllButAxisIter, ArrayIter
 from pypy.interpreter.argument import Arguments
 
 
@@ -190,23 +190,64 @@
         source_state = source_iter.next(source_state)
     return target
 
-reduce_driver = jit.JitDriver(name='numpy_reduce',
-                              greens = ['shapelen', 'func', 'done_func',
-                                        'calc_dtype'],
-                              reds = 'auto')
 
-def compute_reduce(space, obj, calc_dtype, func, done_func, identity):
-    obj_iter, obj_state = obj.create_iter()
+def split_iter(arr, axis_flags):
+    """Prepare 2 iterators for nested iteration over `arr`.
+
+    Arguments:
+        arr: instance of BaseConcreteArray
+        axis_flags: list of bools, one for each dimension of `arr`.The inner
+        iterator operates over the dimensions for which the flag is True
+    """
+    shape = arr.get_shape()
+    strides = arr.get_strides()
+    backstrides = arr.get_backstrides()
+    shapelen = len(shape)
+    assert len(axis_flags) == shapelen
+    inner_shape = [-1] * shapelen
+    inner_strides = [-1] * shapelen
+    inner_backstrides = [-1] * shapelen
+    outer_shape = [-1] * shapelen
+    outer_strides = [-1] * shapelen
+    outer_backstrides = [-1] * shapelen
+    for i in range(len(shape)):
+        if axis_flags[i]:
+            inner_shape[i] = shape[i]
+            inner_strides[i] = strides[i]
+            inner_backstrides[i] = backstrides[i]
+            outer_shape[i] = 1
+            outer_strides[i] = 0
+            outer_backstrides[i] = 0
+        else:
+            outer_shape[i] = shape[i]
+            outer_strides[i] = strides[i]
+            outer_backstrides[i] = backstrides[i]
+            inner_shape[i] = 1
+            inner_strides[i] = 0
+            inner_backstrides[i] = 0
+    inner_iter = ArrayIter(arr, support.product(inner_shape),
+                           inner_shape, inner_strides, inner_backstrides)
+    outer_iter = ArrayIter(arr, support.product(outer_shape),
+                           outer_shape, outer_strides, outer_backstrides)
+    return inner_iter, outer_iter
+
+
+reduce_flat_driver = jit.JitDriver(
+    name='numpy_reduce_flat',
+    greens = ['shapelen', 'func', 'done_func', 'calc_dtype'], reds = 'auto')
+
+def reduce_flat(space, func, w_arr, calc_dtype, done_func, identity):
+    obj_iter, obj_state = w_arr.create_iter()
     if identity is None:
         cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         obj_state = obj_iter.next(obj_state)
     else:
         cur_value = identity.convert_to(space, calc_dtype)
-    shapelen = len(obj.get_shape())
+    shapelen = len(w_arr.get_shape())
     while not obj_iter.done(obj_state):
-        reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                      done_func=done_func,
-                                      calc_dtype=calc_dtype)
+        reduce_flat_driver.jit_merge_point(
+            shapelen=shapelen, func=func,
+            done_func=done_func, calc_dtype=calc_dtype)
         rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
         if done_func is not None and done_func(calc_dtype, rval):
             return rval
@@ -214,33 +255,105 @@
         obj_state = obj_iter.next(obj_state)
     return cur_value
 
-reduce_cum_driver = jit.JitDriver(
-    name='numpy_reduce_cum_driver',
+
+reduce_driver = jit.JitDriver(
+    name='numpy_reduce',
+    greens=['shapelen', 'func', 'dtype'], reds='auto')
+
+def reduce(space, func, w_arr, axis_flags, dtype, out, identity):
+    out_iter, out_state = out.create_iter()
+    out_iter.track_index = False
+    shape = w_arr.get_shape()
+    shapelen = len(shape)
+    inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags)
+    assert outer_iter.size == out_iter.size
+
+    if identity is not None:
+        identity = identity.convert_to(space, dtype)
+    outer_state = outer_iter.reset()
+    while not outer_iter.done(outer_state):
+        inner_state = inner_iter.reset()
+        inner_state.offset = outer_state.offset
+        if identity is not None:
+            w_val = identity
+        else:
+            w_val = inner_iter.getitem(inner_state).convert_to(space, dtype)
+            inner_state = inner_iter.next(inner_state)
+        while not inner_iter.done(inner_state):
+            reduce_driver.jit_merge_point(
+                shapelen=shapelen, func=func, dtype=dtype)
+            w_item = inner_iter.getitem(inner_state).convert_to(space, dtype)
+            w_val = func(dtype, w_item, w_val)
+            inner_state = inner_iter.next(inner_state)
+        out_iter.setitem(out_state, w_val)
+        out_state = out_iter.next(out_state)
+        outer_state = outer_iter.next(outer_state)
+    return out
+
+accumulate_flat_driver = jit.JitDriver(
+    name='numpy_accumulate_flat',
     greens=['shapelen', 'func', 'dtype', 'out_dtype'],
     reds='auto')
 
-def compute_reduce_cumulative(space, obj, out, calc_dtype, func, identity):
-    obj_iter, obj_state = obj.create_iter()
-    out_iter, out_state = out.create_iter()
+def accumulate_flat(space, func, w_arr, calc_dtype, w_out, identity):
+    arr_iter, arr_state = w_arr.create_iter()
+    out_iter, out_state = w_out.create_iter()
     out_iter.track_index = False
     if identity is None:
-        cur_value = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
+        cur_value = arr_iter.getitem(arr_state).convert_to(space, calc_dtype)
         out_iter.setitem(out_state, cur_value)
         out_state = out_iter.next(out_state)
-        obj_state = obj_iter.next(obj_state)
+        arr_state = arr_iter.next(arr_state)
     else:
         cur_value = identity.convert_to(space, calc_dtype)
-    shapelen = len(obj.get_shape())
-    out_dtype = out.get_dtype()
-    while not obj_iter.done(obj_state):
-        reduce_cum_driver.jit_merge_point(
-            shapelen=shapelen, func=func,
-            dtype=calc_dtype, out_dtype=out_dtype)
-        rval = obj_iter.getitem(obj_state).convert_to(space, calc_dtype)
-        cur_value = func(calc_dtype, cur_value, rval)
+    shapelen = len(w_arr.get_shape())
+    out_dtype = w_out.get_dtype()
+    while not arr_iter.done(arr_state):
+        accumulate_flat_driver.jit_merge_point(
+            shapelen=shapelen, func=func, dtype=calc_dtype,
+            out_dtype=out_dtype)
+        w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype)
+        cur_value = func(calc_dtype, cur_value, w_item)
         out_iter.setitem(out_state, out_dtype.coerce(space, cur_value))
         out_state = out_iter.next(out_state)
-        obj_state = obj_iter.next(obj_state)
+        arr_state = arr_iter.next(arr_state)
+
+accumulate_driver = jit.JitDriver(
+    name='numpy_accumulate',
+    greens=['shapelen', 'func', 'calc_dtype'], reds='auto')
+
+
+def accumulate(space, func, w_arr, axis, calc_dtype, w_out, identity):
+    out_iter, out_state = w_out.create_iter()
+    arr_shape = w_arr.get_shape()
+    temp_shape = arr_shape[:axis] + arr_shape[axis + 1:]
+    temp = W_NDimArray.from_shape(space, temp_shape, calc_dtype, w_instance=w_arr)
+    temp_iter = AxisIter(temp.implementation, w_arr.get_shape(), axis)
+    temp_state = temp_iter.reset()
+    arr_iter, arr_state = w_arr.create_iter()
+    arr_iter.track_index = False
+    if identity is not None:
+        identity = identity.convert_to(space, calc_dtype)
+    shapelen = len(arr_shape)
+    while not out_iter.done(out_state):
+        accumulate_driver.jit_merge_point(shapelen=shapelen, func=func,
+                                          calc_dtype=calc_dtype)
+        w_item = arr_iter.getitem(arr_state).convert_to(space, calc_dtype)
+        arr_state = arr_iter.next(arr_state)
+
+        out_indices = out_iter.indices(out_state)
+        if out_indices[axis] == 0:
+            if identity is not None:
+                w_item = func(calc_dtype, identity, w_item)
+        else:
+            cur_value = temp_iter.getitem(temp_state)
+            w_item = func(calc_dtype, cur_value, w_item)
+
+        out_iter.setitem(out_state, w_item)
+        out_state = out_iter.next(out_state)
+        temp_iter.setitem(temp_state, w_item)
+        temp_state = temp_iter.next(temp_state)
+    return w_out
 
 def fill(arr, box):
     arr_iter, arr_state = arr.create_iter()
@@ -298,64 +411,56 @@
             state = x_state
     return out
 
-axis_reduce_driver = jit.JitDriver(name='numpy_axis_reduce',
-                                   greens=['shapelen', 'func', 'dtype'],
-                                   reds='auto')
-
-def do_axis_reduce(space, shape, func, arr, dtype, axis, out, identity, cumulative,
-                   temp):
-    out_iter = AxisIter(out.implementation, arr.get_shape(), axis, cumulative)
-    out_state = out_iter.reset()
-    if cumulative:
-        temp_iter = AxisIter(temp.implementation, arr.get_shape(), axis, False)
-        temp_state = temp_iter.reset()
-    else:
-        temp_iter = out_iter  # hack
-        temp_state = out_state
-    arr_iter, arr_state = arr.create_iter()
-    arr_iter.track_index = False
-    if identity is not None:
-        identity = identity.convert_to(space, dtype)
-    shapelen = len(shape)
-    while not out_iter.done(out_state):
-        axis_reduce_driver.jit_merge_point(shapelen=shapelen, func=func,
-                                           dtype=dtype)
-        w_val = arr_iter.getitem(arr_state).convert_to(space, dtype)
-        arr_state = arr_iter.next(arr_state)
-
-        out_indices = out_iter.indices(out_state)
-        if out_indices[axis] == 0:
-            if identity is not None:
-                w_val = func(dtype, identity, w_val)
-        else:
-            cur = temp_iter.getitem(temp_state)
-            w_val = func(dtype, cur, w_val)
-
-        out_iter.setitem(out_state, w_val)
-        out_state = out_iter.next(out_state)
-        if cumulative:
-            temp_iter.setitem(temp_state, w_val)
-            temp_state = temp_iter.next(temp_state)
-        else:
-            temp_state = out_state
-    return out
-
 
 def _new_argmin_argmax(op_name):
     arg_driver = jit.JitDriver(name='numpy_' + op_name,
                                greens = ['shapelen', 'dtype'],
                                reds = 'auto')
+    arg_flat_driver = jit.JitDriver(name='numpy_flat_' + op_name,
+                                    greens = ['shapelen', 'dtype'],
+                                    reds = 'auto')
 
-    def argmin_argmax(arr):
+    def argmin_argmax(space, w_arr, w_out, axis):
+        from pypy.module.micronumpy.descriptor import get_dtype_cache
+        dtype = w_arr.get_dtype()
+        shapelen = len(w_arr.get_shape())
+        axis_flags = [False] * shapelen
+        axis_flags[axis] = True
+        inner_iter, outer_iter = split_iter(w_arr.implementation, axis_flags)
+        outer_state = outer_iter.reset()
+        out_iter, out_state = w_out.create_iter()
+        while not outer_iter.done(outer_state):
+            inner_state = inner_iter.reset()
+            inner_state.offset = outer_state.offset
+            cur_best = inner_iter.getitem(inner_state)
+            inner_state = inner_iter.next(inner_state)
+            result = 0
+            idx = 1
+            while not inner_iter.done(inner_state):
+                arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+                w_val = inner_iter.getitem(inner_state)
+                new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
+                if dtype.itemtype.ne(new_best, cur_best):
+                    result = idx
+                    cur_best = new_best
+                inner_state = inner_iter.next(inner_state)
+                idx += 1
+            result = get_dtype_cache(space).w_longdtype.box(result)
+            out_iter.setitem(out_state, result)
+            out_state = out_iter.next(out_state)
+            outer_state = outer_iter.next(outer_state)
+        return w_out
+
+    def argmin_argmax_flat(w_arr):
         result = 0
         idx = 1
-        dtype = arr.get_dtype()
-        iter, state = arr.create_iter()
+        dtype = w_arr.get_dtype()
+        iter, state = w_arr.create_iter()
         cur_best = iter.getitem(state)
         state = iter.next(state)
-        shapelen = len(arr.get_shape())
+        shapelen = len(w_arr.get_shape())
         while not iter.done(state):
-            arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
+            arg_flat_driver.jit_merge_point(shapelen=shapelen, dtype=dtype)
             w_val = iter.getitem(state)
             new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val)
             if dtype.itemtype.ne(new_best, cur_best):
@@ -364,9 +469,10 @@
             state = iter.next(state)
             idx += 1
         return result
-    return argmin_argmax
-argmin = _new_argmin_argmax('min')
-argmax = _new_argmin_argmax('max')
+
+    return argmin_argmax, argmin_argmax_flat
+argmin, argmin_flat = _new_argmin_argmax('min')
+argmax, argmax_flat = _new_argmin_argmax('max')
 
 dot_driver = jit.JitDriver(name = 'numpy_dot',
                            greens = ['dtype'],
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -23,6 +23,8 @@
     get_shape_from_iterable, shape_agreement, shape_agreement_multiple,
     is_c_contiguous, is_f_contiguous, calc_strides, new_view)
 from pypy.module.micronumpy.casting import can_cast_array
+from pypy.module.micronumpy.descriptor import get_dtype_cache
+
 
 
 def _match_dot_shapes(space, left, right):
@@ -245,6 +247,20 @@
         if space.is_w(w_idx, space.w_Ellipsis):
             self.implementation.setslice(space, convert_to_array(space, w_value))
             return
+        # TODO: multiarray/mapping.c calls a subclass's __getitem__ here, which
+        # is a big performance hit but necessary for the matrix class. The original
+        # C code is like:
+        #/*
+        #* WARNING: There is a huge special case here. If this is not a
+        #*          base class array, we have to get the view through its
+        #*          very own index machinery.
+        #*          Many subclasses should probably call __setitem__
+        #*          with a base class ndarray view to avoid this.
+        #*/
+        #else if (!(index_type & (HAS_FANCY | HAS_SCALAR_ARRAY))
+        #        && !PyArray_CheckExact(self)) {
+        #view = (PyArrayObject *)PyObject_GetItem((PyObject *)self, ind);
+
         elif isinstance(w_idx, W_NDimArray) and w_idx.get_dtype().is_bool() \
                 and w_idx.ndims() > 0:
             self.setitem_filter(space, w_idx, convert_to_array(space, w_value))

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:43 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix merge.
Message-ID: <20150817121543.A3BA51C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79012:aea1e4e8b77f
Date: 2015-08-17 02:50 +0200
http://bitbucket.org/pypy/pypy/changeset/aea1e4e8b77f/

Log:	Fix merge.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -172,7 +172,7 @@
         try:
             w_globals = space.newdict(module=True)
             space.setitem(w_globals, space.wrap('__builtins__'),
-                          space.builtin_modules['__builtin__'])
+                          space.builtin_modules['builtins'])
             space.setitem(w_globals, space.wrap('c_argument'),
                           space.wrap(c_argument))
             space.appexec([space.wrap(source), w_globals], """(src, glob):

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:45 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:45 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150817121545.B2AC31C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79013:11baca0eafcf
Date: 2015-08-17 02:50 +0200
http://bitbucket.org/pypy/pypy/changeset/11baca0eafcf/

Log:	hg merge py3k

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -172,7 +172,7 @@
         try:
             w_globals = space.newdict(module=True)
             space.setitem(w_globals, space.wrap('__builtins__'),
-                          space.builtin_modules['__builtin__'])
+                          space.builtin_modules['builtins'])
             space.setitem(w_globals, space.wrap('c_argument'),
                           space.wrap(c_argument))
             space.appexec([space.wrap(source), w_globals], """(src, glob):

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:47 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:47 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix merge.
Message-ID: <20150817121547.BB45E1C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79014:3703d41b2c96
Date: 2015-08-17 03:02 +0200
http://bitbucket.org/pypy/pypy/changeset/3703d41b2c96/

Log:	Fix merge.

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -181,7 +181,7 @@
                 if not hasattr(sys, '_pypy_execute_source'):
                     sys._pypy_execute_source = []
                 sys._pypy_execute_source.append(glob)
-                exec stmt in glob
+                exec(stmt, glob)
             """)
         except OperationError, e:
             debug("OperationError:")

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:49 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:49 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150817121549.CA16B1C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79015:7454e904e49c
Date: 2015-08-17 03:02 +0200
http://bitbucket.org/pypy/pypy/changeset/7454e904e49c/

Log:	hg merge py3k

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -181,7 +181,7 @@
                 if not hasattr(sys, '_pypy_execute_source'):
                     sys._pypy_execute_source = []
                 sys._pypy_execute_source.append(glob)
-                exec stmt in glob
+                exec(stmt, glob)
             """)
         except OperationError, e:
             debug("OperationError:")

From noreply at buildbot.pypy.org  Mon Aug 17 14:15:51 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:15:51 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Change time.sleep() to raise ValueError
 for negative sleep lengths.
Message-ID: <20150817121551.D2FD81C130B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79016:72ed98241f4d
Date: 2015-08-17 14:14 +0200
http://bitbucket.org/pypy/pypy/changeset/72ed98241f4d/

Log:	Change time.sleep() to raise ValueError for negative sleep lengths.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -329,8 +329,8 @@
     @unwrap_spec(secs=float)
     def sleep(space, secs):
         if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("sleep length must be non-negative"))
         pytime.sleep(secs)
 else:
     from rpython.rlib import rwin32
@@ -352,8 +352,8 @@
     @unwrap_spec(secs=float)
     def sleep(space, secs):
         if secs < 0:
-            raise OperationError(space.w_IOError,
-                                 space.wrap("Invalid argument: negative time in sleep"))
+            raise OperationError(space.w_ValueError,
+                                 space.wrap("sleep length must be non-negative"))
         # as decreed by Guido, only the main thread can be
         # interrupted.
         main_thread = space.fromcache(State).main_thread
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -18,7 +18,7 @@
         import time
         raises(TypeError, time.sleep, "foo")
         time.sleep(0.12345)
-        raises(IOError, time.sleep, -1.0)
+        raises(ValueError, time.sleep, -1.0)
 
     def test_clock(self):
         import time

From noreply at buildbot.pypy.org  Mon Aug 17 14:29:50 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 14:29:50 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Let time.clock_* raise correct subclasses
	of OSError.
Message-ID: <20150817122950.3BD2B1C1370@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79017:8294dfc3733a
Date: 2015-08-17 14:29 +0200
http://bitbucket.org/pypy/pypy/changeset/8294dfc3733a/

Log:	Let time.clock_* raise correct subclasses of OSError.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -1,6 +1,6 @@
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
-from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror
+from pypy.interpreter.error import OperationError, oefmt, strerror as _strerror, exception_from_saved_errno
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask
@@ -621,8 +621,7 @@
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(clk_id, timespec)
             if ret != 0:
-                raise OperationError(space.w_OSError,
-                                     space.wrap(_get_error_msg()))
+                raise exception_from_saved_errno(space, space.w_OSError)
             result = (float(rffi.getintfield(timespec, 'c_tv_sec')) +
                       float(rffi.getintfield(timespec, 'c_tv_nsec')) * 1e-9)
         return space.wrap(result)
@@ -635,16 +634,14 @@
             rffi.setintfield(timespec, 'c_tv_nsec', int(frac * 1e9))
             ret = c_clock_settime(clk_id, timespec)
             if ret != 0:
-                raise OperationError(space.w_OSError,
-                                     space.wrap(_get_error_msg()))
+                raise exception_from_saved_errno(space, space.w_OSError)
 
     @unwrap_spec(clk_id='c_int')
     def clock_getres(space, clk_id):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_getres(clk_id, timespec)
             if ret != 0:
-                raise OperationError(space.w_OSError,
-                                     space.wrap(_get_error_msg()))
+                raise exception_from_saved_errno(space, space.w_OSError)
             result = (float(rffi.getintfield(timespec, 'c_tv_sec')) +
                       float(rffi.getintfield(timespec, 'c_tv_nsec')) * 1e-9)
         return space.wrap(result)

From noreply at buildbot.pypy.org  Mon Aug 17 15:06:07 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 15:06:07 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Call gc.collect() after each test and
 after having opened many files in CPython's test_bz2.py.
Message-ID: <20150817130607.E459A1C13C0@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79018:abb67a3131df
Date: 2015-08-17 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/abb67a3131df/

Log:	Call gc.collect() after each test and after having opened many files
	in CPython's test_bz2.py.

diff --git a/lib-python/3/test/test_bz2.py b/lib-python/3/test/test_bz2.py
--- a/lib-python/3/test/test_bz2.py
+++ b/lib-python/3/test/test_bz2.py
@@ -54,6 +54,7 @@
         self.filename = TESTFN
 
     def tearDown(self):
+        support.gc_collect()
         if os.path.isfile(self.filename):
             os.unlink(self.filename)
 
@@ -452,6 +453,8 @@
         for i in range(10000):
             o = BZ2File(self.filename)
             del o
+            if i % 100 == 0:
+                support.gc_collect()
 
     def testOpenNonexistent(self):
         self.assertRaises(IOError, BZ2File, "/non/existent")

From noreply at buildbot.pypy.org  Mon Aug 17 16:55:52 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 16:55:52 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: avoid synchronizing operation in
	smallmalloc fastpath
Message-ID: <20150817145552.36F961C130B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1937:f3b1d74a2633
Date: 2015-08-17 15:35 +0200
http://bitbucket.org/pypy/stmgc/changeset/f3b1d74a2633/

Log:	avoid synchronizing operation in smallmalloc fastpath

	Probably doesn't matter much, but change the accounting of allocated
	memory to page instead of (small) obj granularity. In a sense, this
	is also correct and avoids a synchronizing instruction in the
	fastpath of smallmalloc-allocate.

diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -28,7 +28,7 @@
     uint64_t ta = __sync_add_and_fetch(&pages_ctl.total_allocated,
                                        add_or_remove);
 
-    if (ta >= pages_ctl.total_allocated_bound)
+    if (UNLIKELY(ta >= pages_ctl.total_allocated_bound))
         pages_ctl.major_collection_requested = true;
 
     return ta;
diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -126,12 +126,15 @@
        Maybe we can pick one from free_uniform_pages.
      */
     smallpage = free_uniform_pages;
-    if (smallpage != NULL) {
+    if (LIKELY(smallpage != NULL)) {
         if (UNLIKELY(!__sync_bool_compare_and_swap(&free_uniform_pages,
                                                    smallpage,
                                                    smallpage->nextpage)))
             goto retry;
 
+        /* got a new page: */
+        increment_total_allocated(4096);
+
         /* Succeeded: we have a page in 'smallpage', which is not
            initialized so far, apart from the 'nextpage' field read
            above.  Initialize it.
@@ -174,8 +177,6 @@
 
     struct small_free_loc_s *result = *fl;
 
-    increment_total_allocated(size);
-
     if (UNLIKELY(result == NULL)) {
         char *addr = _allocate_small_slowpath(size);
         ((struct object_s*)addr)->stm_flags = 0;
@@ -270,7 +271,6 @@
         }
         else if (!_smallmalloc_sweep_keep(p)) {
             /* the location should be freed now */
-            increment_total_allocated(-szword*8);
 #ifdef STM_TESTS
             /* fill location with 0xdd in all segs except seg0 */
             int j;
@@ -311,6 +311,9 @@
 
         ((struct small_free_loc_s *)baseptr)->nextpage = free_uniform_pages;
         free_uniform_pages = (struct small_free_loc_s *)baseptr;
+
+        /* gave the page back */
+        increment_total_allocated(-4096);
     }
     else if (!any_object_dying) {
         get_fpsz(baseptr)->sz = szword;

From noreply at buildbot.pypy.org  Mon Aug 17 16:55:54 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 16:55:54 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: renaming for clarity
Message-ID: <20150817145554.26B1A1C130B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1938:8a3e95080098
Date: 2015-08-17 16:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/8a3e95080098/

Log:	renaming for clarity

diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -8,9 +8,9 @@
 
 typedef struct {
     uint8_t sz;
-} fpsz_t;
+} full_page_size_t;
 
-static fpsz_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
+static full_page_size_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
 /* ^^^ This array contains the size (in number of words) of the objects
    in the given page, provided it's a "full page of small objects".  It
    is 0 if it's not such a page, if it's fully free, or if it's in
@@ -19,7 +19,7 @@
    technically full yet, it will be very soon in this case).
 */
 
-static fpsz_t *get_fpsz(char *smallpage)
+static full_page_size_t *get_full_page_size(char *smallpage)
 {
     uintptr_t pagenum = (((char *)smallpage) - END_NURSERY_PAGE * 4096UL - stm_object_pages) / 4096;
     /* <= PAGE_SMSIZE_END because we may ask for it when there is no
@@ -118,7 +118,7 @@
 
         /* Succeeded: we have a page in 'smallpage' */
         *fl = smallpage->next;
-        get_fpsz((char *)smallpage)->sz = n;
+        get_full_page_size((char *)smallpage)->sz = n;
         return (char *)smallpage;
     }
 
@@ -156,7 +156,7 @@
         *previous = NULL;
 
         /* The first slot is immediately returned */
-        get_fpsz((char *)smallpage)->sz = n;
+        get_full_page_size((char *)smallpage)->sz = n;
         return (char *)smallpage;
     }
 
@@ -300,6 +300,7 @@
             any_object_remaining = true;
         }
     }
+
     if (!any_object_remaining) {
         /* give page back to free_uniform_pages and thus make it
            inaccessible from all other segments again (except seg0) */
@@ -316,7 +317,9 @@
         increment_total_allocated(-4096);
     }
     else if (!any_object_dying) {
-        get_fpsz(baseptr)->sz = szword;
+        /* this is still a full page. only in this case we set the
+           full_page_size again: */
+        get_full_page_size(baseptr)->sz = szword;
     }
     else {
         check_order_inside_small_page(page_free);
@@ -342,9 +345,9 @@
             if (*fl != NULL) {
                 /* the entry in full_pages_object_size[] should already be
                    szword.  We reset it to 0. */
-                fpsz_t *fpsz = get_fpsz((char *)*fl);
-                assert(fpsz->sz == szword);
-                fpsz->sz = 0;
+                full_page_size_t *full_page_size = get_full_page_size((char *)*fl);
+                assert(full_page_size->sz == szword);
+                full_page_size->sz = 0;
                 sweep_small_page(getbaseptr(*fl), *fl, szword);
                 *fl = NULL;
             }
@@ -354,7 +357,7 @@
         while (page != NULL) {
             /* for every page in small_page_lists: assert that the
                corresponding full_pages_object_size[] entry is 0 */
-            assert(get_fpsz((char *)page)->sz == 0);
+            assert(get_full_page_size((char *)page)->sz == 0);
             nextpage = page->nextpage;
             sweep_small_page(getbaseptr(page), page, szword);
             page = nextpage;
@@ -364,10 +367,10 @@
     /* process the really full pages, which are the ones which still
        have a non-zero full_pages_object_size[] entry */
     char *pageptr = uninitialized_page_stop;
-    fpsz_t *fpsz_start = get_fpsz(pageptr);
-    fpsz_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
-                                               PAGE_SMSIZE_START];
-    fpsz_t *fpsz;
+    full_page_size_t *fpsz_start = get_full_page_size(pageptr);
+    full_page_size_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
+                                                         PAGE_SMSIZE_START];
+    full_page_size_t *fpsz;
     for (fpsz = fpsz_start; fpsz < fpsz_end; fpsz++, pageptr += 4096) {
         uint8_t sz = fpsz->sz;
         if (sz != 0) {

From noreply at buildbot.pypy.org  Mon Aug 17 16:55:56 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 16:55:56 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: Merge with use-gcc
Message-ID: <20150817145556.198CA1C130B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1939:916134a837e9
Date: 2015-08-17 16:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/916134a837e9/

Log:	Merge with use-gcc

diff --git a/c8/stm/pages.c b/c8/stm/pages.c
--- a/c8/stm/pages.c
+++ b/c8/stm/pages.c
@@ -28,7 +28,7 @@
     uint64_t ta = __sync_add_and_fetch(&pages_ctl.total_allocated,
                                        add_or_remove);
 
-    if (ta >= pages_ctl.total_allocated_bound)
+    if (UNLIKELY(ta >= pages_ctl.total_allocated_bound))
         pages_ctl.major_collection_requested = true;
 
     return ta;
diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -8,9 +8,9 @@
 
 typedef struct {
     uint8_t sz;
-} fpsz_t;
+} full_page_size_t;
 
-static fpsz_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
+static full_page_size_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
 /* ^^^ This array contains the size (in number of words) of the objects
    in the given page, provided it's a "full page of small objects".  It
    is 0 if it's not such a page, if it's fully free, or if it's in
@@ -19,7 +19,7 @@
    technically full yet, it will be very soon in this case).
 */
 
-static fpsz_t *get_fpsz(char *smallpage)
+static full_page_size_t *get_full_page_size(char *smallpage)
 {
     uintptr_t pagenum = (((char *)smallpage) - END_NURSERY_PAGE * 4096UL - stm_object_pages) / 4096;
     /* <= PAGE_SMSIZE_END because we may ask for it when there is no
@@ -118,7 +118,7 @@
 
         /* Succeeded: we have a page in 'smallpage' */
         *fl = smallpage->next;
-        get_fpsz((char *)smallpage)->sz = n;
+        get_full_page_size((char *)smallpage)->sz = n;
         return (char *)smallpage;
     }
 
@@ -126,12 +126,15 @@
        Maybe we can pick one from free_uniform_pages.
      */
     smallpage = free_uniform_pages;
-    if (smallpage != NULL) {
+    if (LIKELY(smallpage != NULL)) {
         if (UNLIKELY(!__sync_bool_compare_and_swap(&free_uniform_pages,
                                                    smallpage,
                                                    smallpage->nextpage)))
             goto retry;
 
+        /* got a new page: */
+        increment_total_allocated(4096);
+
         /* Succeeded: we have a page in 'smallpage', which is not
            initialized so far, apart from the 'nextpage' field read
            above.  Initialize it.
@@ -153,7 +156,7 @@
         *previous = NULL;
 
         /* The first slot is immediately returned */
-        get_fpsz((char *)smallpage)->sz = n;
+        get_full_page_size((char *)smallpage)->sz = n;
         return (char *)smallpage;
     }
 
@@ -174,8 +177,6 @@
 
     struct small_free_loc_s *result = *fl;
 
-    increment_total_allocated(size);
-
     if (UNLIKELY(result == NULL)) {
         char *addr = _allocate_small_slowpath(size);
         ((struct object_s*)addr)->stm_flags = 0;
@@ -270,7 +271,6 @@
         }
         else if (!_smallmalloc_sweep_keep(p)) {
             /* the location should be freed now */
-            increment_total_allocated(-szword*8);
 #ifdef STM_TESTS
             /* fill location with 0xdd in all segs except seg0 */
             int j;
@@ -300,6 +300,7 @@
             any_object_remaining = true;
         }
     }
+
     if (!any_object_remaining) {
         /* give page back to free_uniform_pages and thus make it
            inaccessible from all other segments again (except seg0) */
@@ -311,9 +312,14 @@
 
         ((struct small_free_loc_s *)baseptr)->nextpage = free_uniform_pages;
         free_uniform_pages = (struct small_free_loc_s *)baseptr;
+
+        /* gave the page back */
+        increment_total_allocated(-4096);
     }
     else if (!any_object_dying) {
-        get_fpsz(baseptr)->sz = szword;
+        /* this is still a full page. only in this case we set the
+           full_page_size again: */
+        get_full_page_size(baseptr)->sz = szword;
     }
     else {
         check_order_inside_small_page(page_free);
@@ -339,9 +345,9 @@
             if (*fl != NULL) {
                 /* the entry in full_pages_object_size[] should already be
                    szword.  We reset it to 0. */
-                fpsz_t *fpsz = get_fpsz((char *)*fl);
-                assert(fpsz->sz == szword);
-                fpsz->sz = 0;
+                full_page_size_t *full_page_size = get_full_page_size((char *)*fl);
+                assert(full_page_size->sz == szword);
+                full_page_size->sz = 0;
                 sweep_small_page(getbaseptr(*fl), *fl, szword);
                 *fl = NULL;
             }
@@ -351,7 +357,7 @@
         while (page != NULL) {
             /* for every page in small_page_lists: assert that the
                corresponding full_pages_object_size[] entry is 0 */
-            assert(get_fpsz((char *)page)->sz == 0);
+            assert(get_full_page_size((char *)page)->sz == 0);
             nextpage = page->nextpage;
             sweep_small_page(getbaseptr(page), page, szword);
             page = nextpage;
@@ -361,10 +367,10 @@
     /* process the really full pages, which are the ones which still
        have a non-zero full_pages_object_size[] entry */
     char *pageptr = uninitialized_page_stop;
-    fpsz_t *fpsz_start = get_fpsz(pageptr);
-    fpsz_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
-                                               PAGE_SMSIZE_START];
-    fpsz_t *fpsz;
+    full_page_size_t *fpsz_start = get_full_page_size(pageptr);
+    full_page_size_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
+                                                         PAGE_SMSIZE_START];
+    full_page_size_t *fpsz;
     for (fpsz = fpsz_start; fpsz < fpsz_end; fpsz++, pageptr += 4096) {
         uint8_t sz = fpsz->sz;
         if (sz != 0) {

From noreply at buildbot.pypy.org  Mon Aug 17 16:55:58 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 16:55:58 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: reuse smallobj pages
 only after they are half-empty again
Message-ID: <20150817145558.0C7341C130B@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1940:01da8e5d803d
Date: 2015-08-17 16:59 +0200
http://bitbucket.org/pypy/stmgc/changeset/01da8e5d803d/

Log:	reuse smallobj pages only after they are half-empty again

	Again trying to improve the range-merging chance - again failing.
	Probably need a completely different strategy if we can improve this
	at all. And I should first find more meaningful benchmarks where the
	problem really shows up.

diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -36,6 +36,7 @@
 static void teardown_smallmalloc(void)
 {
     memset(small_page_lists, 0, sizeof(small_page_lists));
+    memset(pretty_full_pages, 0, sizeof(pretty_full_pages));
     assert(free_uniform_pages == NULL);   /* done by the previous line */
     first_small_uniform_loc = (uintptr_t) -1;
 #ifdef STM_TESTS
@@ -261,6 +262,7 @@
        to have it ignored in the end because we put the page into
        'free_uniform_pages' */
 
+    long free_slots_balance = 0;
     for (i = 0; i <= 4096 - size; i += size) {
         char *p = baseptr + i;
         if (p == (char *)fl) {
@@ -268,9 +270,11 @@
             flprev = fl;
             fl = fl->next;
             any_object_dying = true;
+            free_slots_balance++;
         }
         else if (!_smallmalloc_sweep_keep(p)) {
             /* the location should be freed now */
+            free_slots_balance++;
 #ifdef STM_TESTS
             /* fill location with 0xdd in all segs except seg0 */
             int j;
@@ -297,6 +301,7 @@
         }
         else {
             //dprintf(("keep small %p : %lu\n", (char*)(p - stm_object_pages), szword*8));
+            free_slots_balance--;
             any_object_remaining = true;
         }
     }
@@ -322,9 +327,15 @@
         get_full_page_size(baseptr)->sz = szword;
     }
     else {
-        check_order_inside_small_page(page_free);
-        page_free->nextpage = small_page_lists[szword];
-        small_page_lists[szword] = page_free;
+        if (free_slots_balance < 0) {
+            /* "pretty full" */
+            page_free->nextpage = pretty_full_pages[szword];
+            pretty_full_pages[szword] = page_free;
+        } else {
+            check_order_inside_small_page(page_free);
+            page_free->nextpage = small_page_lists[szword];
+            small_page_lists[szword] = page_free;
+        }
     }
 }
 
@@ -333,8 +344,10 @@
     long i, szword;
     for (szword = 2; szword < GC_N_SMALL_REQUESTS; szword++) {
         struct small_free_loc_s *page = small_page_lists[szword];
+        struct small_free_loc_s *pfpage = pretty_full_pages[szword];
         struct small_free_loc_s *nextpage;
         small_page_lists[szword] = NULL;
+        pretty_full_pages[szword] = NULL;
 
         /* process the pages that the various segments are busy filling */
         /* including sharing seg0 for old-malloced things */
@@ -362,6 +375,14 @@
             sweep_small_page(getbaseptr(page), page, szword);
             page = nextpage;
         }
+
+        /* process all the pretty full pages */
+        while (pfpage != NULL) {
+            assert(get_full_page_size((char *)pfpage)->sz == 0);
+            nextpage = pfpage->nextpage;
+            sweep_small_page(getbaseptr(pfpage), pfpage, szword);
+            pfpage = nextpage;
+        }
     }
 
     /* process the really full pages, which are the ones which still
diff --git a/c8/stm/smallmalloc.h b/c8/stm/smallmalloc.h
--- a/c8/stm/smallmalloc.h
+++ b/c8/stm/smallmalloc.h
@@ -38,6 +38,11 @@
 
 #define free_uniform_pages   (small_page_lists[0])
 
+/* here are pages that should be in small_page_lists, but since they
+   are "pretty full", let's use them again only after they are less
+   full (XXX: or if we run out of pages). */
+static struct small_free_loc_s *pretty_full_pages[GC_N_SMALL_REQUESTS];
+
 
 /* For is_small_uniform(). */
 static uintptr_t first_small_uniform_loc = (uintptr_t) -1;

From noreply at buildbot.pypy.org  Mon Aug 17 17:49:49 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 17:49:49 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: also try to merge
 fragments in the normal _synchronize_fragment
Message-ID: <20150817154949.187981C1370@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1941:9bf05d9105aa
Date: 2015-08-17 17:44 +0200
http://bitbucket.org/pypy/stmgc/changeset/9bf05d9105aa/

Log:	also try to merge fragments in the normal _synchronize_fragment

	Actually, PyPy shows much higher ratio of successful merges than
	duhton. Still, it's hard to make out any improvement in the noise.

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1727,11 +1727,30 @@
        of the fragment need syncing to other segments? (keep privatization
        lock until the "flush") */
 
-    /* Enqueue this object (or fragemnt of object) */
-    if (STM_PSEGMENT->sq_len == SYNC_QUEUE_SIZE)
+    int sq_len = STM_PSEGMENT->sq_len;
+    /* try to merge with previous fragment: */
+    int min = sq_len-4 <= 0 ? 0 : sq_len-4; /* go back 4 elems */
+    for (int i = sq_len-1; i >= min; i--) {
+        stm_char *start = STM_PSEGMENT->sq_fragments[i];
+        ssize_t size = STM_PSEGMENT->sq_fragsizes[i];
+
+        if (start + size == frag) {
+            /* merge! */
+            if ((size + frag_size) + ((uintptr_t)start & 4095) > 4096)
+                break;      /* doesn't fit inside the same page */
+
+            STM_PSEGMENT->sq_fragsizes[i] = size + frag_size;
+            return;
+        }
+    }
+
+    /* Enqueue this object (or fragment of object) */
+    if (sq_len == SYNC_QUEUE_SIZE) {
         synchronize_objects_flush();
-    STM_PSEGMENT->sq_fragments[STM_PSEGMENT->sq_len] = frag;
-    STM_PSEGMENT->sq_fragsizes[STM_PSEGMENT->sq_len] = frag_size;
+        sq_len = STM_PSEGMENT->sq_len;
+    }
+    STM_PSEGMENT->sq_fragments[sq_len] = frag;
+    STM_PSEGMENT->sq_fragsizes[sq_len] = frag_size;
     ++STM_PSEGMENT->sq_len;
 }
 
@@ -1748,6 +1767,7 @@
     OPT_ASSERT(obj_size >= 16);
 
     if (LIKELY(is_small_uniform(obj))) {
+        /* XXX: could also use the knowledge of full_pages_object_size ^^^ */
         assert(!(obj->stm_flags & GCFLAG_CARDS_SET));
         OPT_ASSERT(obj_size <= GC_LAST_SMALL_SIZE);
         _synchronize_fragment((stm_char *)obj, obj_size);
@@ -1828,7 +1848,8 @@
         stm_char *obj_start = (stm_char*)obj;
         long i;
         long min = lst->count - 4 * 2; /* go back 4 elems */
-        for (i = lst->count - 2; i >= min && i >= 0; i -= 2) {
+        min = min >= 0 ? min : 0;
+        for (i = lst->count - 2; i >= min; i -= 2) {
             stm_char *start = (stm_char*)lst->items[i];
             ssize_t size = (ssize_t)lst->items[i+1];
 
@@ -1837,11 +1858,13 @@
                 if ((size + obj_size) + ((uintptr_t)start & 4095) > 4096)
                     break;      /* doesn't fit inside the same page */
 
+                //fprintf(stderr, "merged\n");
                 lst->items[i+1] = size + obj_size;
                 return;
             }
         }
     }
+    //fprintf(stderr, "nomerge\n");
 
     /* no merge was found */
     STM_PSEGMENT->small_overflow_obj_ranges[obj_size / 8] =

From noreply at buildbot.pypy.org  Mon Aug 17 17:49:51 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 17 Aug 2015 17:49:51 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: fix tests for f3b1d74a2633
Message-ID: <20150817154951.1B1581C1370@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1942:25e30d665282
Date: 2015-08-17 17:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/25e30d665282/

Log:	fix tests for f3b1d74a2633

diff --git a/c8/test/test_gcpage.py b/c8/test/test_gcpage.py
--- a/c8/test/test_gcpage.py
+++ b/c8/test/test_gcpage.py
@@ -325,9 +325,10 @@
         actual_big = (big + 15 ) & ~15
 
         self.start_transaction()
-        assert lib._stm_total_allocated() == 64 + (actual_big + LMO) # large malloc'd
+        # 4096 for 1 page of smallmalloc:
+        assert lib._stm_total_allocated() == 4096 + (actual_big + LMO) # large malloc'd
         stm_major_collect()
-        assert lib._stm_total_allocated() == 64 + (actual_big + LMO) # large malloc'd
+        assert lib._stm_total_allocated() == 4096 + (actual_big + LMO) # large malloc'd
         self.commit_transaction()
 
     def test_bug(self):
@@ -376,19 +377,19 @@
         assert lib._stm_total_allocated() == 0
         self.push_root(new)
         stm_minor_collect()
-        assert lib._stm_total_allocated() == 16
+        assert lib._stm_total_allocated() == 4096
 
         new = self.pop_root()
         assert not is_in_nursery(new)
         stm_minor_collect()
-        assert lib._stm_total_allocated() == 16
+        assert lib._stm_total_allocated() == 4096
 
         stm_major_collect()
         assert lib._stm_total_allocated() == 0
 
     def test_mixed_major_collections(self):
         import random
-        obj_sizes = [16, 48, 1024, 1000*8]
+        obj_sizes = [1024, 1000*8]
 
         self.start_transaction()
         random.seed(123)
@@ -398,11 +399,7 @@
         NOBJS = 100
         for _ in range(NOBJS):
             osize = random.choice(obj_sizes)
-            is_small = osize <= GC_LAST_SMALL_SIZE
-            if is_small:
-                allocated += osize
-            else:
-                allocated += osize + LMO
+            allocated += osize + LMO
 
             o = stm_allocate(osize)
             self.push_root(o)

From noreply at buildbot.pypy.org  Mon Aug 17 19:09:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Aug 2015 19:09:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: Rename files and functions to split
 some parts from RPython
Message-ID: <20150817170932.2FC331C1193@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79019:bbcfcccf2f68
Date: 2015-08-17 16:59 +0200
http://bitbucket.org/pypy/pypy/changeset/bbcfcccf2f68/

Log:	Rename files and functions to split some parts from RPython

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,24 +40,18 @@
         **eci_kwds))
 
 
-    vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP,
+    vmprof_init = rffi.llexternal("vmprof_init", [rffi.INT], rffi.CCHARP,
                                   compilation_info=eci)
-    vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT,
+    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.LONG], rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT,
+    vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
                                      compilation_info=eci,
                                      save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf",
+    vmprof_write_buf = rffi.llexternal("vmprof_write_buf",
                                        [rffi.CCHARP, rffi.LONG],
                                        lltype.Void, compilation_info=eci)
-
-    ## vmprof_register_virtual_function = rffi.llexternal(
-    ##     "vmprof_register_virtual_function",
-    ##     [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    ##     compilation_info=eci, _nowrapper=True)
-
-    vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals",
+    vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals",
                                             [rffi.INT], lltype.Void,
                                             compilation_info=eci)
     return CInterface(locals())
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -1,22 +1,3 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in
- * gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *   Armin Rigo - arigo at tunes.org
- *
- */
-
 #define _GNU_SOURCE 1
 
 
@@ -39,431 +20,4 @@
 #endif
 
 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
-
-
-/************************************************************/
-
-// functions copied from libunwind using dlopen
-
-static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-static int (*unw_step)(unw_cursor_t*) = NULL;
-static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static int profile_file = -1;
-
-
-RPY_EXTERN
-char *rpython_vmprof_init(int fd)
-{
-    if (!unw_get_reg) {
-        void *libhandle;
-
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
-            goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
-            goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
-            goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
-            goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
-            goto error;
-    }
-    if (prepare_concurrent_bufs() < 0)
-        return "out of memory";
-
-    assert(fd >= 0);
-    profile_file = fd;
-    return NULL;
-
- error:
-    return dlerror();
-}
-
-/************************************************************/
-
-/* value: last bit is 1 if signals must be ignored; all other bits
-   are a counter for how many threads are currently in a signal handler */
-static long volatile signal_handler_value = 1;
-
-RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
-{
-    if (!ignored) {
-        __sync_fetch_and_and(&signal_handler_value, ~1L);
-    }
-    else {
-        /* set the last bit, and wait until concurrently-running signal
-           handlers finish */
-        while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
-            usleep(1);
-        }
-    }
-}
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH   \
-    ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-struct prof_stacktrace_s {
-    char padding[sizeof(long) - 1];
-    char marker;
-    long count, depth;
-    void *stack[];
-};
-
-static long profile_interval_usec = 0;
-static char atfork_hook_installed = 0;
-
-
-/* ******************************************************
- * libunwind workaround for process JIT frames correctly
- * ******************************************************
- */
-
-#include "rvmprof_get_custom_offset.h"
-
-typedef struct {
-    void* _unused1;
-    void* _unused2;
-    void* sp;
-    void* ip;
-    void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4];
-} vmprof_hacked_unw_cursor_t;
-
-static int vmprof_unw_step(unw_cursor_t *cp, int first_run)
-{
-    void* ip;
-    void* sp;
-    ptrdiff_t sp_offset;
-    unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip);
-    unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
-    if (!first_run) {
-        // make sure we're pointing to the CALL and not to the first
-        // instruction after. If the callee adjusts the stack for us
-        // it's not safe to be at the instruction after
-        ip -= 1;
-    }
-    sp_offset = vmprof_unw_get_custom_offset(ip, cp);
-
-    if (sp_offset == -1) {
-        // it means that the ip is NOT in JITted code, so we can use the
-        // stardard unw_step
-        return unw_step(cp);
-    }
-    else {
-        // this is a horrible hack to manually walk the stack frame, by
-        // setting the IP and SP in the cursor
-        vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp;
-        void* bp = (void*)sp + sp_offset;
-        cp2->sp = bp;
-        bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext)
-{
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    (void)ret;
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) {
-            // found main loop stack frame
-            void* sp;
-            unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-            void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
-            void **arg_ptr = (void**)arg_addr;
-            /* if (mainloop_get_virtual_ip) {
-               ip = mainloop_get_virtual_ip(*arg_ptr);
-               } else { */
-            ip = *arg_ptr;
-        }
-
-        int first_run = (n == 0);
-        result[n++] = ip;
-        n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0)
-            break;
-    }
-    return n;
-}
-
-
-/* *************************************************************
- * the signal handler
- * *************************************************************
- */
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
-{
-    long val = __sync_fetch_and_add(&signal_handler_value, 2L);
-
-    if ((val & 1) == 0) {
-        int saved_errno = errno;
-        int fd = profile_file;
-        assert(fd >= 0);
-
-        struct profbuf_s *p = reserve_buffer(fd);
-        if (p == NULL) {
-            /* ignore this signal: there are no free buffers right now */
-        }
-        else {
-            int depth;
-            struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
-            st->marker = MARKER_STACKTRACE;
-            st->count = 1;
-            st->stack[0] = GetPC((ucontext_t*)ucontext);
-            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
-            depth++;  // To account for pc value in stack[0];
-            st->depth = depth;
-            p->data_offset = offsetof(struct prof_stacktrace_s, marker);
-            p->data_size = (depth * sizeof(void *) +
-                            sizeof(struct prof_stacktrace_s) -
-                            offsetof(struct prof_stacktrace_s, marker));
-            commit_buffer(fd, p);
-        }
-
-        errno = saved_errno;
-    }
-
-    __sync_sub_and_fetch(&signal_handler_value, 2L);
-}
-
-
-/* *************************************************************
- * the setup and teardown functions
- * *************************************************************
- */
-
-static int install_sigprof_handler(void)
-{
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-        sigaction(SIGPROF, &sa, NULL) == -1)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_handler(void)
-{
-    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
-        return -1;
-    return 0;
-}
-
-static int install_sigprof_timer(void)
-{
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = profile_interval_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (profile_interval_usec > 0) {
-        remove_sigprof_timer();
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (profile_interval_usec > 0) {
-        install_sigprof_timer();
-    }
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
-{
-    assert(profile_file >= 0);
-    assert(interval_usec > 0);
-    profile_interval_usec = interval_usec;
-
-    if (install_pthread_atfork_hooks() == -1)
-        goto error;
-    if (install_sigprof_handler() == -1)
-        goto error;
-    if (install_sigprof_timer() == -1)
-        goto error;
-    rpython_vmprof_ignore_signals(0);
-    return 0;
-
- error:
-    profile_file = -1;
-    profile_interval_usec = 0;
-    return -1;
-}
-
-static int _write_all(const void *buf, size_t bufsize)
-{
-    while (bufsize > 0) {
-        ssize_t count = write(profile_file, buf, bufsize);
-        if (count <= 0)
-            return -1;   /* failed */
-        buf += count;
-        bufsize -= count;
-    }
-    return 0;
-}
-
-static int close_profile(void)
-{
-    char buf[4096];
-    ssize_t size;
-    unsigned char marker = MARKER_TRAILER;
-
-    if (_write_all(&marker, 1) < 0)
-        return -1;
-
-#ifdef __linux__
-    // copy /proc/self/maps to the end of the profile file
-    int srcfd = open("/proc/self/maps", O_RDONLY);
-    if (srcfd < 0)
-        return -1;
-
-    while ((size = read(srcfd, buf, sizeof buf)) > 0) {
-        if (_write_all(buf, size) < 0) {
-            close(srcfd);
-            return -1;
-        }
-    }
-    close(srcfd);
-#else
-    // freebsd and mac
-    sprintf(buf, "procstat -v %d", getpid());
-    FILE *srcf = popen(buf, "r");
-    if (!srcf)
-        return -1;
-
-    while ((size = fread(buf, 1, sizeof buf, src))) {
-        if (_write_all(buf, size) < 0) {
-            pclose(srcf);
-            return -1;
-        }
-    }
-    pclose(srcf);
-#endif
-
-    /* don't close() the file descriptor from here */
-    profile_file = -1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_disable(void)
-{
-    rpython_vmprof_ignore_signals(1);
-    profile_interval_usec = 0;
-
-    if (remove_sigprof_timer() == -1)
-        return -1;
-    if (remove_sigprof_handler() == -1)
-        return -1;
-    if (shutdown_concurrent_bufs(profile_file) < 0)
-        return -1;
-    return close_profile();
-}
-
-RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
-{
-    struct profbuf_s *p;
-
-    while ((p = reserve_buffer(profile_file)) == NULL) {
-        /* spin loop waiting for a buffer to be ready; should almost never
-           be the case */
-        usleep(1);
-    }
-
-    if (size > SINGLE_BUF_SIZE)
-        size = SINGLE_BUF_SIZE;
-    memcpy(p->data, buf, size);
-    p->data_size = size;
-
-    commit_buffer(profile_file, p);
-}
+#include "vmprof_main.h"
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
 
-RPY_EXTERN char *rpython_vmprof_init(int);
-RPY_EXTERN void rpython_vmprof_ignore_signals(int);
-RPY_EXTERN int rpython_vmprof_enable(long);
-RPY_EXTERN int rpython_vmprof_disable(void);
-RPY_EXTERN void rpython_vmprof_write_buf(char *, long);
+RPY_EXTERN char *vmprof_init(int);
+RPY_EXTERN void vmprof_ignore_signals(int);
+RPY_EXTERN int vmprof_enable(long);
+RPY_EXTERN int vmprof_disable(void);
+RPY_EXTERN void vmprof_write_buf(char *, long);
diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
rename from rpython/rlib/rvmprof/src/rvmprof_config.h
rename to rpython/rlib/rvmprof/src/vmprof_config.h
diff --git a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
rename from rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
rename to rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
diff --git a/rpython/rlib/rvmprof/src/rvmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
rename from rpython/rlib/rvmprof/src/rvmprof_getpc.h
rename to rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/rvmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -44,7 +44,7 @@
 #ifndef BASE_GETPC_H_
 #define BASE_GETPC_H_
 
-#include "rvmprof_config.h"
+#include "vmprof_config.h"
 
 // On many linux systems, we may need _GNU_SOURCE to get access to
 // the defined constants that define the register we want to see (eg
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/vmprof_main.h
copy from rpython/rlib/rvmprof/src/rvmprof.c
copy to rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -19,26 +19,6 @@
 
 #define _GNU_SOURCE 1
 
-
-#ifdef RPYTHON_LL2CTYPES
-   /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
-#  ifndef RPY_EXTERN
-#    define RPY_EXTERN RPY_EXPORTED
-#  endif
-#  define RPY_EXPORTED  extern __attribute__((visibility("default")))
-#  define VMPROF_ADDR_OF_TRAMPOLINE(addr)  0
-
-#else
-
-#  include "common_header.h"
-#  include "rvmprof.h"
-#  ifndef VMPROF_ADDR_OF_TRAMPOLINE
-#   error "RPython program using rvmprof, but not calling vmprof_execute_code()"
-#  endif
-
-#endif
-
-
 #include 
 #include 
 #include 
@@ -49,9 +29,9 @@
 #include 
 #include 
 #include 
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
+#include "vmprof_getpc.h"
+#include "vmprof_unwind.h"
+#include "vmprof_mt.h"
 
 
 /************************************************************/
@@ -67,7 +47,7 @@
 
 
 RPY_EXTERN
-char *rpython_vmprof_init(int fd)
+char *vmprof_init(int fd)
 {
     if (!unw_get_reg) {
         void *libhandle;
@@ -101,7 +81,7 @@
 static long volatile signal_handler_value = 1;
 
 RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
+void vmprof_ignore_signals(int ignored)
 {
     if (!ignored) {
         __sync_fetch_and_and(&signal_handler_value, ~1L);
@@ -145,7 +125,7 @@
  * ******************************************************
  */
 
-#include "rvmprof_get_custom_offset.h"
+#include "vmprof_get_custom_offset.h"
 
 typedef struct {
     void* _unused1;
@@ -358,7 +338,7 @@
 }
 
 RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
+int vmprof_enable(long interval_usec)
 {
     assert(profile_file >= 0);
     assert(interval_usec > 0);
@@ -370,7 +350,7 @@
         goto error;
     if (install_sigprof_timer() == -1)
         goto error;
-    rpython_vmprof_ignore_signals(0);
+    vmprof_ignore_signals(0);
     return 0;
 
  error:
@@ -435,9 +415,9 @@
 }
 
 RPY_EXTERN
-int rpython_vmprof_disable(void)
+int vmprof_disable(void)
 {
-    rpython_vmprof_ignore_signals(1);
+    vmprof_ignore_signals(1);
     profile_interval_usec = 0;
 
     if (remove_sigprof_timer() == -1)
@@ -450,7 +430,7 @@
 }
 
 RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
+void vmprof_write_buf(char *buf, long size)
 {
     struct profbuf_s *p;
 
diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
rename from rpython/rlib/rvmprof/src/rvmprof_mt.h
rename to rpython/rlib/rvmprof/src/vmprof_mt.h
--- a/rpython/rlib/rvmprof/src/rvmprof_mt.h
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -193,7 +193,7 @@
 static int shutdown_concurrent_bufs(int fd)
 {
     /* no signal handler can be running concurrently here, because we
-       already did rpython_vmprof_ignore_signals(1) */
+       already did vmprof_ignore_signals(1) */
     assert(profbuf_write_lock == 0);
     profbuf_write_lock = 2;
 
diff --git a/rpython/rlib/rvmprof/src/rvmprof_unwind.h b/rpython/rlib/rvmprof/src/vmprof_unwind.h
rename from rpython/rlib/rvmprof/src/rvmprof_unwind.h
rename to rpython/rlib/rvmprof/src/vmprof_unwind.h

From noreply at buildbot.pypy.org  Mon Aug 17 19:09:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Aug 2015 19:09:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move to C the _write_header() code: it
 was not guaranteed (but only very
Message-ID: <20150817170934.365181C1193@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79020:71fc12ecb018
Date: 2015-08-17 18:15 +0200
http://bitbucket.org/pypy/pypy/changeset/71fc12ecb018/

Log:	Move to C the _write_header() code: it was not guaranteed (but only
	very likely) to occur as the first write. Moreover, this is easier
	to keep the code identical with CPython.

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,9 +40,10 @@
         **eci_kwds))
 
 
-    vmprof_init = rffi.llexternal("vmprof_init", [rffi.INT], rffi.CCHARP,
-                                  compilation_info=eci)
-    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.LONG], rffi.INT,
+    vmprof_init = rffi.llexternal("vmprof_init",
+                                  [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+                                  rffi.CCHARP, compilation_info=eci)
+    vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
     vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -102,18 +102,14 @@
         assert fileno >= 0
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
-        if not (1e-6 <= interval < 1.0):
-            raise VMProfError("bad value for 'interval'")
-        interval_usec = int(interval * 1000000.0)
 
-        p_error = self.cintf.vmprof_init(fileno)
+        p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
         self.fileno = fileno
-        self._write_header(interval_usec)
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable(interval_usec)
+        res = self.cintf.vmprof_enable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
@@ -154,19 +150,6 @@
         # a maximum of 8184 bytes.  This should be guaranteed here because:
         assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184
 
-    def _write_header(self, interval_usec):
-        b = StringBuilder()
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(3, b)
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(interval_usec, b)
-        _write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
-        buf = b.build()
-        self.cintf.vmprof_write_buf(buf, len(buf))
-
 
 def _write_long_to_string_builder(l, b):
     b.append(chr(l & 0xff))
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
 
-RPY_EXTERN char *vmprof_init(int);
+RPY_EXTERN char *vmprof_init(int, double, const char *);
 RPY_EXTERN void vmprof_ignore_signals(int);
-RPY_EXTERN int vmprof_enable(long);
+RPY_EXTERN int vmprof_enable(void);
 RPY_EXTERN int vmprof_disable(void);
 RPY_EXTERN void vmprof_write_buf(char *, long);
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -44,11 +44,17 @@
 static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
 
 static int profile_file = -1;
+static long prepare_interval_usec;
 
+static int opened_profile(const char *interp_name);
 
 RPY_EXTERN
-char *vmprof_init(int fd)
+char *vmprof_init(int fd, double interval, const char *interp_name)
 {
+    if (interval < 1e-6 || interval >= 1.0)
+        return "bad value for 'interval'";
+    prepare_interval_usec = (int)(interval * 1000000.0);
+
     if (!unw_get_reg) {
         void *libhandle;
 
@@ -68,6 +74,10 @@
 
     assert(fd >= 0);
     profile_file = fd;
+    if (opened_profile(interp_name) < 0) {
+        profile_file = -1;
+        return strerror(errno);
+    }
     return NULL;
 
  error:
@@ -108,6 +118,7 @@
 #define MARKER_STACKTRACE '\x01'
 #define MARKER_VIRTUAL_IP '\x02'
 #define MARKER_TRAILER '\x03'
+#define MARKER_INTERP_NAME '\x04'
 
 struct prof_stacktrace_s {
     char padding[sizeof(long) - 1];
@@ -338,11 +349,11 @@
 }
 
 RPY_EXTERN
-int vmprof_enable(long interval_usec)
+int vmprof_enable(void)
 {
     assert(profile_file >= 0);
-    assert(interval_usec > 0);
-    profile_interval_usec = interval_usec;
+    assert(prepare_interval_usec > 0);
+    profile_interval_usec = prepare_interval_usec;
 
     if (install_pthread_atfork_hooks() == -1)
         goto error;
@@ -371,6 +382,27 @@
     return 0;
 }
 
+static int opened_profile(const char *interp_name)
+{
+    struct {
+        long hdr[5];
+        char interp_name[257];
+    } header;
+
+    size_t namelen = strlen(interp_name);
+    assert(namelen <= 255);
+
+    header.hdr[0] = 0;
+    header.hdr[1] = 3;
+    header.hdr[2] = 0;
+    header.hdr[3] = prepare_interval_usec;
+    header.hdr[4] = 0;
+    header.interp_name[0] = MARKER_INTERP_NAME;
+    header.interp_name[1] = namelen;
+    memcpy(&header.interp_name[2], interp_name, namelen);
+    return _write_all(&header, 5 * sizeof(long) + 2 + namelen);
+}
+
 static int close_profile(void)
 {
     char buf[4096];

From noreply at buildbot.pypy.org  Mon Aug 17 19:09:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Aug 2015 19:09:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: Thread support
Message-ID: <20150817170936.3B32A1C1193@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79021:3c7aa1d4efa6
Date: 2015-08-17 18:38 +0200
http://bitbucket.org/pypy/pypy/changeset/3c7aa1d4efa6/

Log:	Thread support

diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -118,7 +118,11 @@
 #define MARKER_STACKTRACE '\x01'
 #define MARKER_VIRTUAL_IP '\x02'
 #define MARKER_TRAILER '\x03'
-#define MARKER_INTERP_NAME '\x04'
+#define MARKER_INTERP_NAME '\x04'   /* deprecated */
+#define MARKER_HEADER '\x05'
+
+#define VERSION_BASE '\x00'
+#define VERSION_THREAD_ID '\x01'
 
 struct prof_stacktrace_s {
     char padding[sizeof(long) - 1];
@@ -228,6 +232,23 @@
     return n;
 }
 
+static void *get_current_thread_id(void)
+{
+    /* xxx This function is a hack on two fronts:
+
+       - It assumes that pthread_self() is async-signal-safe.  This
+         should be true on Linux.  I hope it is also true elsewhere.
+
+       - It abuses pthread_self() by assuming it just returns an
+         integer.  According to comments in CPython's source code, the
+         platforms where it is not the case are rare nowadays.
+
+       An alternative would be to try to look if the information is
+       available in the ucontext_t in the caller.
+    */
+    return (void *)pthread_self();
+}
+
 
 /* *************************************************************
  * the signal handler
@@ -253,8 +274,9 @@
             st->marker = MARKER_STACKTRACE;
             st->count = 1;
             st->stack[0] = GetPC((ucontext_t*)ucontext);
-            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
+            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-2, ucontext);
             depth++;  // To account for pc value in stack[0];
+            st->stack[depth++] = get_current_thread_id();
             st->depth = depth;
             p->data_offset = offsetof(struct prof_stacktrace_s, marker);
             p->data_size = (depth * sizeof(void *) +
@@ -386,7 +408,7 @@
 {
     struct {
         long hdr[5];
-        char interp_name[257];
+        char interp_name[259];
     } header;
 
     size_t namelen = strlen(interp_name);
@@ -397,10 +419,12 @@
     header.hdr[2] = 0;
     header.hdr[3] = prepare_interval_usec;
     header.hdr[4] = 0;
-    header.interp_name[0] = MARKER_INTERP_NAME;
-    header.interp_name[1] = namelen;
-    memcpy(&header.interp_name[2], interp_name, namelen);
-    return _write_all(&header, 5 * sizeof(long) + 2 + namelen);
+    header.interp_name[0] = MARKER_HEADER;
+    header.interp_name[1] = '\x00';
+    header.interp_name[2] = VERSION_THREAD_ID;
+    header.interp_name[3] = namelen;
+    memcpy(&header.interp_name[4], interp_name, namelen);
+    return _write_all(&header, 5 * sizeof(long) + 4 + namelen);
 }
 
 static int close_profile(void)

From noreply at buildbot.pypy.org  Mon Aug 17 19:17:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 17 Aug 2015 19:17:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: Translation fix
Message-ID: <20150817171712.602251C1193@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79022:559b4a0a9100
Date: 2015-08-17 18:19 +0100
http://bitbucket.org/pypy/pypy/changeset/559b4a0a9100/

Log:	Translation fix

diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -6,9 +6,9 @@
 #endif
 
 #ifdef RPYTHON_VMPROF
-RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored);
+RPY_EXTERN void vmprof_ignore_signals(int ignored);
 static void pypy_codemap_invalid_set(int ignored) {
-    rpython_vmprof_ignore_signals(ignored);
+    vmprof_ignore_signals(ignored);
 }
 #else
 static void pypy_codemap_invalid_set(int ignored) {

From noreply at buildbot.pypy.org  Mon Aug 17 19:53:15 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 19:53:15 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Re-add this Py_INCREF,
 which was removed in this branch for some reason.
Message-ID: <20150817175315.451DD1C0170@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79023:68c0a4532d83
Date: 2015-08-17 19:52 +0200
http://bitbucket.org/pypy/pypy/changeset/68c0a4532d83/

Log:	Re-add this Py_INCREF, which was removed in this branch for some
	reason.

diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -1330,6 +1330,7 @@
         return NULL;
 
     obj = PyUnicode_FromString("test");
+    Py_INCREF(obj);
     PyTuple_SET_ITEM(tuple, 0, obj);
     Py_INCREF(Py_None);
     PyTuple_SET_ITEM(tuple, 1, Py_None);

From noreply at buildbot.pypy.org  Mon Aug 17 20:05:01 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Mon, 17 Aug 2015 20:05:01 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Back out changeset 68c0a4532d83. The
 _testcapimodule.c was actually copied from CPython.
Message-ID: <20150817180501.3CB711C0170@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79024:769da83a3ec7
Date: 2015-08-17 20:05 +0200
http://bitbucket.org/pypy/pypy/changeset/769da83a3ec7/

Log:	Back out changeset 68c0a4532d83. The _testcapimodule.c was actually
	copied from CPython.

diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -1330,7 +1330,6 @@
         return NULL;
 
     obj = PyUnicode_FromString("test");
-    Py_INCREF(obj);
     PyTuple_SET_ITEM(tuple, 0, obj);
     Py_INCREF(Py_None);
     PyTuple_SET_ITEM(tuple, 1, Py_None);

From noreply at buildbot.pypy.org  Mon Aug 17 22:01:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 17 Aug 2015 22:01:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix the short preamble deal,
 not 100% just yet, but getting there
Message-ID: <20150817200113.1C7B31C136F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79025:a6e577ab0c1a
Date: 2015-08-17 22:00 +0200
http://bitbucket.org/pypy/pypy/changeset/a6e577ab0c1a/

Log:	fix the short preamble deal, not 100% just yet, but getting there

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -246,8 +246,7 @@
                                descr=start_descr)
     mid_descr_token = TargetToken(jitcell_token,
                                   original_jitcell_token=jitcell_token)
-    mid_label = ResOperation(rop.LABEL, start_state.end_args +
-                             loop_info.extra_label_args,
+    mid_label = ResOperation(rop.LABEL, loop_info.label_args,
                              descr=mid_descr_token)
     # XXX assign short preamble and virtual state
     loop_ops[-1].setdescr(mid_descr_token)
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -137,7 +137,7 @@
             v1, v2 = v2, v1
         # if both are constant, the pure optimization will deal with it
         if v2.is_constant() and not v1.is_constant():
-            if not self.optimizer.is_inputarg(arg1):
+            if False and not self.optimizer.is_inputarg(arg1):
                 if arg1.getopnum() == rop.INT_ADD:
                     prod_arg1 = arg1.getarg(0)
                     prod_arg2 = arg1.getarg(1)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -337,6 +337,7 @@
         return op
 
     def is_inputarg(self, op):
+        #return True
         return op in self.inparg_dict
 
     def get_constant_box(self, box):
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -198,7 +198,7 @@
         assert not invented_name
 
     def __repr__(self):
-        return "INP(%r)" % (self.preamble_op,)
+        return "INP(%r -> %r)" % (self.res, self.preamble_op)
 
 class ShortBoxes(object):
     """ This is a container used for creating all the exported short
@@ -209,16 +209,15 @@
         # of AbstractShortOp
         self.potential_ops = OrderedDict()
         self.produced_short_boxes = {}
+        self.short_inputargs = []
         # a way to produce const boxes, e.g. setfield_gc(p0, Const).
         # We need to remember those, but they don't produce any new boxes
         self.const_short_boxes = []
-        label_d = {}
-        for arg in label_args:
-            label_d[arg] = None
-        for box in inputargs:
-            if box in label_d:
-                renamed = OpHelpers.inputarg_from_tp(box.type)
-                self.potential_ops[box] = ShortInputArg(box, renamed)
+        for i in range(len(inputargs)):
+            box = inputargs[i]
+            renamed = OpHelpers.inputarg_from_tp(box.type)
+            self.potential_ops[box] = ShortInputArg(label_args[i], renamed)
+            self.short_inputargs.append(renamed)
 
         optimizer.produce_potential_short_preamble_ops(self)
 
@@ -295,6 +294,7 @@
         return pop
 
     def create_short_inputargs(self, label_args):
+        return self.short_inputargs
         short_inpargs = []
         for i in range(len(label_args)):
             inparg = self.produced_short_boxes.get(label_args[i], None)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8687,5 +8687,36 @@
         """
         self.optimize_loop(ops, expected, preamble)
 
+    def test_loop_variant_mul1(self):
+        ops = """
+        [i3, i4, i5]
+        i6 = int_mul(i5, i5)
+        i7 = int_add(i4, i6)
+        i9 = int_add(i5, 1)
+        i10 = int_mul(i9, i9)
+        i11 = int_add(i7, i10)
+        i13 = int_sub(i3, 1)
+        i15 = int_gt(i13, 0)
+        guard_true(i15) []
+        jump(i13, i11, i9)
+        """
+        expected = """
+        [i8, i7, i5, i6]
+        i10 = int_add(i7, i6)
+        i11 = int_add(i5, 1)
+        i12 = int_mul(i11, i11)
+        i13 = int_add(i10, i12)
+        i14 = int_sub(i8, 1)
+        i15 = int_gt(i14, 0)
+        guard_true(i15) []
+        jump(i14, i13, i11, i12)
+        """
+        expected_short = """
+        [i16, i17, i18]
+        i19 = int_mul(i18, i18)
+        jump(i19)
+        """
+        self.optimize_loop(ops, expected, expected_short=expected_short)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -194,6 +194,7 @@
         self.short_preamble_producer = ShortPreambleBuilder(
             exported_state.short_boxes, exported_state.short_inputargs,
             exported_state.exported_infos, self.optimizer)
+
         for produced_op in exported_state.short_boxes:
             produced_op.produce_op(self, exported_state.exported_infos)
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -80,14 +80,14 @@
         self.check_resops({'jump': 1, 'int_gt': 2, 'int_add': 2,
                            'guard_true': 2, 'int_sub': 2})
 
-        if 0 and self.basic:
+        if self.basic:
             found = 0
             for op in get_stats().get_all_loops()[0]._all_operations():
                 if op.getopname() == 'guard_true':
                     liveboxes = op.getfailargs()
                     assert len(liveboxes) == 3
                     for box in liveboxes:
-                        assert isinstance(box, history.BoxInt)
+                        assert box.type == 'i'
                     found += 1
             assert found == 2
 

From noreply at buildbot.pypy.org  Mon Aug 17 22:56:01 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 17 Aug 2015 22:56:01 +0200 (CEST)
Subject: [pypy-commit] buildbot default: whoops
Message-ID: <20150817205601.A13401C1193@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r956:d3f3f5d7e18f
Date: 2015-08-17 23:56 +0300
http://bitbucket.org/pypy/buildbot/changeset/d3f3f5d7e18f/

Log:	whoops

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -671,7 +671,7 @@
             haltOnFailure=True,
             workdir='.'))
         self.addStep(ShellCmd(
-            description="copy cffi import libraries"
+            description="copy cffi import libraries",
             command=['cp', '-rv', 'pypy-c/lib_pypy/*.so', 'build/lib_pypy'],
             haltOnFailure=True,
             workdir='.'))

From noreply at buildbot.pypy.org  Tue Aug 18 00:10:28 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Aug 2015 00:10:28 +0200 (CEST)
Subject: [pypy-commit] buildbot default: directly setting step_status is
 deprecated, use api available for many years now
Message-ID: <20150817221028.F09981C0170@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r957:3edf8f86e63f
Date: 2015-08-18 00:16 +0300
http://bitbucket.org/pypy/buildbot/changeset/3edf8f86e63f/

Log:	directly setting step_status is deprecated, use api available for
	many years now

diff --git a/bot2/pypybuildbot/test/test_builds.py b/bot2/pypybuildbot/test/test_builds.py
--- a/bot2/pypybuildbot/test/test_builds.py
+++ b/bot2/pypybuildbot/test/test_builds.py
@@ -99,7 +99,7 @@
     kw = inst._getStepFactory().kwargs
     rebuilt = factory(**kw)
     rebuilt.build = FakeBuild()
-    rebuilt.step_status = FakeStepStatus()
+    rebuilt.setStepStatus(FakeStepStatus())
     rebuilt.runCommand = lambda *args: FakeDeferred()
     rebuilt.start()
     assert pth.join('mstr').check(dir=True)
@@ -186,7 +186,7 @@
         factory = inst._getStepFactory().factory
         kw = inst._getStepFactory().kwargs
         self.rebuilt = factory(**kw)
-        self.rebuilt.step_status = FakeStepStatus()
+        self.rebuilt.setStepStatus(FakeStepStatus())
         self.rebuilt.deferred = FakeDeferred()
 
     def test_has_revision(self):

From noreply at buildbot.pypy.org  Tue Aug 18 02:09:48 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 18 Aug 2015 02:09:48 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Extract timespec-to-seconds conversion.
Message-ID: <20150818000948.7C9CC1C0170@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79026:ab86ae37dd28
Date: 2015-08-18 00:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ab86ae37dd28/

Log:	Extract timespec-to-seconds conversion.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -616,15 +616,17 @@
     return space.wrap(float(tt))
 
 if _POSIX:
+    def _timespec_to_seconds(timespec):
+        return int(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9
+
     @unwrap_spec(clk_id='c_int')
     def clock_gettime(space, clk_id):
         with lltype.scoped_alloc(TIMESPEC) as timespec:
             ret = c_clock_gettime(clk_id, timespec)
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
-            result = (float(rffi.getintfield(timespec, 'c_tv_sec')) +
-                      float(rffi.getintfield(timespec, 'c_tv_nsec')) * 1e-9)
-        return space.wrap(result)
+            secs = _timespec_to_seconds(timespec)
+        return space.wrap(secs)
 
     @unwrap_spec(clk_id='c_int', secs=float)
     def clock_settime(space, clk_id, secs):
@@ -642,9 +644,8 @@
             ret = c_clock_getres(clk_id, timespec)
             if ret != 0:
                 raise exception_from_saved_errno(space, space.w_OSError)
-            result = (float(rffi.getintfield(timespec, 'c_tv_sec')) +
-                      float(rffi.getintfield(timespec, 'c_tv_nsec')) * 1e-9)
-        return space.wrap(result)
+            secs = _timespec_to_seconds(timespec)
+        return space.wrap(secs)
 
     def tzset(space):
         """tzset()

From noreply at buildbot.pypy.org  Tue Aug 18 02:09:50 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 18 Aug 2015 02:09:50 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Check for availability of clock_gettime()
 instead of assuming it exists on posix systems.
Message-ID: <20150818000950.B0C581C0170@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79027:0c1306f0e901
Date: 2015-08-18 01:01 +0200
http://bitbucket.org/pypy/pypy/changeset/0c1306f0e901/

Log:	Check for availability of clock_gettime() instead of assuming it
	exists on posix systems.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -4,7 +4,7 @@
 from pypy.interpreter.gateway import unwrap_spec
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rlib.rarithmetic import intmask
-from rpython.rlib.rtime import c_clock_gettime, TIMESPEC, win_perf_counter
+from rpython.rlib.rtime import win_perf_counter
 from rpython.rlib import rposix
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 import math
@@ -114,6 +114,7 @@
     CLOCKS_PER_SEC = platform.ConstantInteger("CLOCKS_PER_SEC")
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
+    has_clock_gettime = platform.Has('clock_gettime')
 
 CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
                    'CLOCK_PROCESS_CPUTIME_ID', 'CLOCK_REALTIME',
@@ -192,13 +193,15 @@
 c_mktime = external('mktime', [TM_P], rffi.TIME_T)
 c_localtime = external('localtime', [rffi.TIME_TP], TM_P,
                        save_err=rffi.RFFI_SAVE_ERRNO)
-if _POSIX:
+if cConfig.has_clock_gettime:
+    from rpython.rlib.rtime import TIMESPEC, c_clock_gettime
     c_clock_settime = external('clock_settime',
                                [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
                                save_err=rffi.RFFI_SAVE_ERRNO)
     c_clock_getres = external('clock_getres',
                               [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT,
                               save_err=rffi.RFFI_SAVE_ERRNO)
+if _POSIX:
     c_tzset = external('tzset', [], lltype.Void)
 if _WIN:
     win_eci = ExternalCompilationInfo(
@@ -615,7 +618,7 @@
 
     return space.wrap(float(tt))
 
-if _POSIX:
+if cConfig.has_clock_gettime:
     def _timespec_to_seconds(timespec):
         return int(timespec.c_tv_sec) + int(timespec.c_tv_nsec) * 1e-9
 
@@ -647,6 +650,7 @@
             secs = _timespec_to_seconds(timespec)
         return space.wrap(secs)
 
+if _POSIX:
     def tzset(space):
         """tzset()
 

From noreply at buildbot.pypy.org  Tue Aug 18 02:09:52 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 18 Aug 2015 02:09:52 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Implement time.process_time(). Windows
	support untested.
Message-ID: <20150818000952.E7DF31C0170@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79028:3cb76c96dbc1
Date: 2015-08-18 02:03 +0200
http://bitbucket.org/pypy/pypy/changeset/3cb76c96dbc1/

Log:	Implement time.process_time(). Windows support untested.

diff --git a/pypy/module/time/__init__.py b/pypy/module/time/__init__.py
--- a/pypy/module/time/__init__.py
+++ b/pypy/module/time/__init__.py
@@ -21,6 +21,7 @@
         '_STRUCT_TM_ITEMS': 'space.wrap(interp_time._STRUCT_TM_ITEMS)',
         'monotonic': 'interp_time.monotonic',
         'perf_counter': 'interp_time.perf_counter',
+        'process_time': 'interp_time.process_time',
     }
 
     if os.name == "posix":
diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -115,6 +115,7 @@
     clock_t = platform.SimpleType("clock_t", rffi.ULONG)
     has_gettimeofday = platform.Has('gettimeofday')
     has_clock_gettime = platform.Has('clock_gettime')
+    CLOCK_PROF = platform.DefinedConstantInteger('CLOCK_PROF')
 
 CLOCK_CONSTANTS = ['CLOCK_HIGHRES', 'CLOCK_MONOTONIC', 'CLOCK_MONOTONIC_RAW',
                    'CLOCK_PROCESS_CPUTIME_ID', 'CLOCK_REALTIME',
@@ -762,3 +763,53 @@
 else:
     def perf_counter(space):
         return monotonic(space)
+
+
+if _WIN:
+    # untested so far
+    def process_time(space):
+        process_times = _time.GetProcessTimes(handle)
+        return (process_times['UserTime'] + process_times['KernelTime']) * 1e-7
+
+        from rpython.rlib.rposix import GetCurrentProcess, GetProcessTimes
+        current_process = GetCurrentProcess()
+        with lltype.scoped_alloc(rwin32.FILETIME) as creation_time, \
+             lltype.scoped_alloc(rwin32.FILETIME) as exit_time, \
+             lltype.scoped_alloc(rwin32.FILETIME) as kernel_time, \
+             lltype.scoped_alloc(rwin32.FILETIME) as user_time:
+            GetProcessTimes(current_process, creation_time, exit_time,
+                            kernel_time, user_time)
+            kernel_time2 = (kernel_time.dwLowDateTime |
+                            kernel_time.dwHighDateTime << 32)
+            user_time2 = (user_time.dwLowDateTime |
+                          user_time.dwHighDateTime << 32)
+        return space.wrap((float(kernel_time2) + float(user_time2)) * 1e-7)
+
+else:
+    def process_time(space):
+        if cConfig.has_clock_gettime and (
+                cConfig.CLOCK_PROF is not None or
+                cConfig.CLOCK_PROCESS_CPUTIME_ID is not None):
+            if cConfig.CLOCK_PROF is not None:
+                clk_id = cConfig.CLOCK_PROF
+            else:
+                clk_id = cConfig.CLOCK_PROCESS_CPUTIME_ID
+            with lltype.scoped_alloc(TIMESPEC) as timespec:
+                ret = c_clock_gettime(clk_id, timespec)
+                if ret == 0:
+                    return space.wrap(_timespec_to_seconds(timespec))
+        if True: # XXX available except if it isn't?
+            from rpython.rlib.rtime import (c_getrusage, RUSAGE, RUSAGE_SELF,
+                                            decode_timeval)
+            with lltype.scoped_alloc(RUSAGE) as rusage:
+                ret = c_getrusage(RUSAGE_SELF, rusage)
+                if ret == 0:
+                    return space.wrap(decode_timeval(rusage.c_ru_utime) +
+                                      decode_timeval(rusage.c_ru_stime))
+        if hasattr(rposix, 'c_times'):
+            with lltype.scoped_alloc(rposix.TMS) as tms:
+                ret = rposix.c_times(tms)
+                if ret != -1:
+                    cpu_time = tms.c_tms_utime + tms.c_tms_stime
+                    return space.wrap(cpu_time / rposix.CLOCK_TICKS_PER_SECOND)
+        return clock(space)
diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py
--- a/pypy/module/time/test/test_time.py
+++ b/pypy/module/time/test/test_time.py
@@ -370,3 +370,12 @@
     def test_perf_counter(self):
         import time
         assert isinstance(time.perf_counter(), float)
+
+    def test_process_time(self):
+        import time
+        t1 = time.process_time()
+        assert isinstance(t1, float)
+        time.sleep(0.1)
+        t2 = time.process_time()
+        # process_time() should not include time spent during sleep
+        assert (t2 - t1) < 0.05
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -172,7 +172,7 @@
     RUSAGE_SELF = RUSAGE_SELF or 0
     c_getrusage = external('getrusage', 
                            [rffi.INT, lltype.Ptr(RUSAGE)],
-                           lltype.Void,
+                           rffi.INT,
                            releasegil=False)
 
 def win_perf_counter():

From noreply at buildbot.pypy.org  Tue Aug 18 10:23:25 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 18 Aug 2015 10:23:25 +0200 (CEST)
Subject: [pypy-commit] stmgc c8-faster-smallobj-sync: better comment
Message-ID: <20150818082325.6D63B1C023F@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: c8-faster-smallobj-sync
Changeset: r1943:29753a525a5f
Date: 2015-08-18 10:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/29753a525a5f/

Log:	better comment

	Overall, the performance with this branch is mostly unchanged.
	However, when rtyping 'pixie', we get ~5% improvement (~3% without
	the "pretty full pages" tweak). While I consider these results
	inconclusive, the added complexity is pretty small and there may be
	more tweaks to be done.

diff --git a/c8/stm/smallmalloc.c b/c8/stm/smallmalloc.c
--- a/c8/stm/smallmalloc.c
+++ b/c8/stm/smallmalloc.c
@@ -328,7 +328,8 @@
     }
     else {
         if (free_slots_balance < 0) {
-            /* "pretty full" */
+            /* "pretty full" (>= 50% used), so don't use the page
+               for new allocations until it is at least half-empty again  */
             page_free->nextpage = pretty_full_pages[szword];
             pretty_full_pages[szword] = page_free;
         } else {

From noreply at buildbot.pypy.org  Tue Aug 18 10:41:05 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 10:41:05 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: added a new jit parameter
 vec_params which encode the previously known vectorize_user,
 vec_cost and add a maximum trace length and a ratio in the fast path to skip
 vector traces. renamed vectorize to vec, vectorize_user to vec_all
Message-ID: <20150818084105.ECE4B1C023F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79029:fc65e2cfbacc
Date: 2015-08-18 10:41 +0200
http://bitbucket.org/pypy/pypy/changeset/fc65e2cfbacc/

Log:	added a new jit parameter vec_params which encode the previously
	known vectorize_user, vec_cost and add a maximum trace length and a
	ratio in the fast path to skip vector traces. renamed vectorize to
	vec, vectorize_user to vec_all

diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py
--- a/pypy/module/micronumpy/test/test_zjit.py
+++ b/pypy/module/micronumpy/test/test_zjit.py
@@ -97,7 +97,7 @@
                                              backendopt=True,
                                              graph_and_interp_only=True,
                                              ProfilerClass=Profiler,
-                                             vectorize=True)
+                                             vec=True)
             self.__class__.interp = interp
             self.__class__.graph = graph
 
diff --git a/rpython/jit/backend/x86/test/test_zrpy_vecopt.py b/rpython/jit/backend/x86/test/test_zrpy_vecopt.py
--- a/rpython/jit/backend/x86/test/test_zrpy_vecopt.py
+++ b/rpython/jit/backend/x86/test/test_zrpy_vecopt.py
@@ -17,7 +17,7 @@
     t.buildrtyper().specialize()
 
     if kwds['jit']:
-        apply_jit(t, vectorize=True)
+        apply_jit(t, vec=True)
 
 class TestVecOptX86(object):
     def test_translate(self):
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -69,8 +69,8 @@
         optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts)
         if unroll:
             if not export_state and \
-                ((warmstate.vectorize and jitdriver_sd.vectorize) \
-                 or warmstate.vectorize_user):
+                ((warmstate.vec and jitdriver_sd.vec) \
+                 or warmstate.vec_all):
                 optimize_vector(metainterp_sd, jitdriver_sd, loop,
                                 optimizations, inline_short_preamble,
                                 start_state, warmstate)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -333,14 +333,14 @@
     storedebug = None
 
 class FakeWarmState(object):
-    vectorize = True # default is on
-    vectorize_user = False
+    vec = True # default is on
+    vec_all = False
     vec_cost = 0
     def __init__(self, enable_opts):
         self.enable_opts = enable_opts
 
 class FakeJitDriverStaticData(object):
-    vectorize = False
+    vec = False
 
 class FakeMetaInterpStaticData(object):
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -22,7 +22,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT
 
 class FakeJitDriverStaticData(object):
-    vectorize=True
+    vec=True
 
 class FakeCostModel(CostModel):
     def __init__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -35,6 +35,9 @@
                     inline_short_preamble, start_state, warmstate):
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
+    user_code = not jitdriver_sd.vec and warmstate.vec_all
+    if user_code and user_loop_bail_fast_path(loop, warmstate):
+        return
     version = loop.snapshot()
     try:
         debug_start("vec-opt-loop")
@@ -47,7 +50,6 @@
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
         opt.propagate_all_forward()
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
-        user_code = not jitdriver_sd.vectorize and warmstate.vectorize_user
         gso.propagate_all_forward(opt.loop, user_code)
         # connect all compile loop version fail descriptors to this version
         version.register_all_guards(loop.operations, opt.appended_arg_count)
@@ -85,6 +87,36 @@
         else:
             raise
 
+def user_loop_bail_fast_path(loop, warmstate):
+    """ in a fast path over the trace loop: try to prevent vecopt
+    of spending time on a loop that will most probably fail """
+
+    resop_count = 0 # the count of operations minus debug_merge_points
+    vector_instr = 0
+    at_least_one_array_access = True
+    for i,op in enumerate(loop.operations):
+        if op.getopnum() == rop.DEBUG_MERGE_POINT:
+            continue
+
+        if op.vector >= 0 and not op.is_guard():
+            vector_instr += 1
+
+        resop_count += 1
+
+        if op.is_primitive_array_access():
+            at_least_one_array_access = True
+
+    if not at_least_one_array_access:
+        return True
+
+    if resop_count > warmstate.vec_length:
+        return True
+
+    if float(vector_instr)/float(resop_count) <= warmstate.vec_ratio:
+        return True
+
+    return False
+
 def cmp_pack_lt(a,b):
     return a.left.getindex() < b.left.getindex()
 packsort = listsort.make_timsort_class(lt=cmp_pack_lt)
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -1123,7 +1123,7 @@
 
         if self.metainterp.seen_loop_header_for_jdindex < 0:
             if not any_operation:
-                if jitdriver_sd.vectorize or jitdriver_sd.warmstate.vectorize_user:
+                if jitdriver_sd.vec or jitdriver_sd.warmstate.vec_all:
                     self.metainterp.generate_guard(rop.GUARD_EARLY_EXIT)
                 return
             if self.metainterp.portal_call_depth or not self.metainterp.get_procedure_token(greenboxes, True):
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -51,7 +51,7 @@
 
         trace_limit = sys.maxint
         enable_opts = ALL_OPTS_DICT
-        vectorize = True
+        vec = True
 
     if kwds.pop('disable_optimizations', False):
         FakeWarmRunnerState.enable_opts = {}
@@ -70,7 +70,7 @@
         greenfield_info = None
         result_type = result_kind
         portal_runner_ptr = "???"
-        vectorize = False
+        vec = False
 
     stats = history.Stats()
     cpu = CPUClass(rtyper, stats, None, False)
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -66,7 +66,7 @@
         index = 0
         warmstate = FakeState()
         virtualizable_info = None
-        vectorize = False
+        vec = False
 
 def test_compile_loop():
     cpu = FakeCPU()
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -24,7 +24,7 @@
                               policy=policy,
                               CPUClass=self.CPUClass,
                               type_system=self.type_system,
-                              vectorize=1)
+                              vec=True)
 
     @py.test.mark.parametrize('i',[3,4,5,6,7,8,9,50])
     def test_vectorize_simple_load_arith_store_int_add_index(self,i):
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -32,7 +32,7 @@
 # Bootstrapping
 
 def apply_jit(translator, backend_name="auto", inline=False,
-              vectorize=False, enable_opts=ALL_OPTS_NAMES, **kwds):
+              vec=False, enable_opts=ALL_OPTS_NAMES, **kwds):
     if 'CPUClass' not in kwds:
         from rpython.jit.backend.detect_cpu import getcpuclass
         kwds['CPUClass'] = getcpuclass(backend_name)
@@ -47,7 +47,7 @@
                                     **kwds)
     for jd in warmrunnerdesc.jitdrivers_sd:
         jd.warmstate.set_param_inlining(inline)
-        jd.warmstate.set_param_vectorize(vectorize)
+        jd.warmstate.set_param_vec(vec)
         jd.warmstate.set_param_enable_opts(enable_opts)
     warmrunnerdesc.finish()
     translator.warmrunnerdesc = warmrunnerdesc    # for later debugging
@@ -68,12 +68,11 @@
     return jittify_and_run(interp, graph, args, backendopt=backendopt, **kwds)
 
 def jittify_and_run(interp, graph, args, repeat=1, graph_and_interp_only=False,
-                    backendopt=False, trace_limit=sys.maxint,
-                    inline=False, loop_longevity=0, retrace_limit=5,
-                    function_threshold=4,
+                    backendopt=False, trace_limit=sys.maxint, inline=False,
+                    loop_longevity=0, retrace_limit=5, function_threshold=4,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
-                    max_unroll_recursion=7, vectorize=0, vectorize_user=0,
-                    vec_cost=0, **kwds):
+                    max_unroll_recursion=7, vec=0, vec_params='0:0:50:0.6',
+                    **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -96,9 +95,8 @@
         jd.warmstate.set_param_max_retrace_guards(max_retrace_guards)
         jd.warmstate.set_param_enable_opts(enable_opts)
         jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
-        jd.warmstate.set_param_vectorize(vectorize)
-        jd.warmstate.set_param_vectorize_user(vectorize_user)
-        jd.warmstate.set_param_vec_cost(vec_cost)
+        jd.warmstate.set_param_vec(vec)
+        jd.warmstate.set_param_vec_params(vec_params)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
@@ -398,7 +396,7 @@
         graph.func._dont_inline_ = True
         graph.func._jit_unroll_safe_ = True
         jd.jitdriver = block.operations[pos].args[1].value
-        jd.vectorize = jd.jitdriver.vectorize
+        jd.vec = jd.jitdriver.vec
         jd.portal_runner_ptr = ""
         jd.result_type = history.getkind(jd.portal_graph.getreturnvar()
                                          .concretetype)[0]
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -300,14 +300,21 @@
             if self.warmrunnerdesc.memory_manager:
                 self.warmrunnerdesc.memory_manager.max_unroll_recursion = value
 
-    def set_param_vectorize(self, value):
-        self.vectorize = bool(value)
+    def set_param_vec(self, value):
+        self.vec = bool(value)
 
-    def set_param_vectorize_user(self, value):
-        self.vectorize_user = bool(value)
-
-    def set_param_vec_cost(self, value):
-        self.vec_cost = bool(value)
+    def set_param_vec_params(self, value):
+        values = value.split(":")
+        self.vec_all = bool(values[0])
+        self.vec_cost = 0
+        if len(values) > 1:
+            self.vec_cost = int(values[1])
+        self.vec_length = 50
+        if len(values) > 2:
+            self.vec_length = int(values[2])
+        self.vec_ratio = 0.60
+        if len(values) > 3:
+            self.vec_ratio = float(values[3])
 
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -553,9 +553,14 @@
     'enable_opts': 'INTERNAL USE ONLY (MAY NOT WORK OR LEAD TO CRASHES): '
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
-    'vectorize': 'turn on the vectorization optimization (vecopt). requires sse4.1',
-    'vectorize_user': 'turn on the vecopt for the python user program. requires sse4.1',
-    'vec_cost': 'threshold which traces to vectorize.',
+    'vec': 'turn on the vectorization optimization (vecopt). requires sse4.1',
+    'vec_params': 'parameters to the optimization separated by colons. [:[:[:]]]. '
+                  'all = 1: try to vectorize trace loops that occur outside of the numpy library. '
+                  'cost = 0: threshold for which traces to bail. 0 means the costs '
+                  'balance the unpacking, if below the vectorizer bails out. '
+                  'length = 50:  the amount of instructions allowed in "all" traces. '
+                  'ratio = 0.60: the number statements that have vector equivalents divided '
+                  'by the total number of trace instructions.',
 }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -571,9 +576,8 @@
               'disable_unrolling': 200,
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
-              'vectorize': 0,
-              'vectorize_user': 0,
-              'vec_cost': 0,
+              'vec': 0,
+              'vec_params': '0:0:50:0.60',
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 
@@ -636,7 +640,7 @@
         self.can_never_inline = can_never_inline
         self.should_unroll_one_iteration = should_unroll_one_iteration
         self.check_untranslated = check_untranslated
-        self.vectorize = vectorize
+        self.vec = vectorize
 
     def _freeze_(self):
         return True

From noreply at buildbot.pypy.org  Tue Aug 18 12:07:44 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Aug 2015 12:07:44 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: fix translation
Message-ID: <20150818100744.0BCB21C023F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79030:75ee3e2981e2
Date: 2015-08-17 12:31 +0200
http://bitbucket.org/pypy/pypy/changeset/75ee3e2981e2/

Log:	fix translation

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -145,7 +145,7 @@
         if type(list_w[i]) is not type(w_firstobj):
             break
     else:
-        return cache.get_or_build(type(w_firstobj))
+        return cache.get_or_build(w_firstobj.__class__)
     return cache.generic_strategy
 
 
@@ -958,7 +958,7 @@
             strategy = self.space.fromcache(FloatListStrategy)
         else:
             cache = self.space.fromcache(FixedClsStrategyCache)
-            strategy = cache.get_or_build(type(w_item))
+            strategy = cache.get_or_build(w_item.__class__)
 
         storage = strategy.get_empty_storage(self.get_sizehint())
         w_list.strategy = strategy
@@ -1635,7 +1635,7 @@
     def is_correct_type(self, w_obj):
         cls = self.get_known_cls()
         if cls is not None:
-            return type(w_obj) is cls
+            return w_obj.__class__ is cls
         return True
 
     def list_is_correct_type(self, w_list):

From noreply at buildbot.pypy.org  Tue Aug 18 12:07:46 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Aug 2015 12:07:46 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: remove prints
Message-ID: <20150818100746.0F3C81C023F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79031:3f0a99f295de
Date: 2015-08-18 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/3f0a99f295de/

Log:	remove prints

diff --git a/pypy/interpreter/valueprof.py b/pypy/interpreter/valueprof.py
--- a/pypy/interpreter/valueprof.py
+++ b/pypy/interpreter/valueprof.py
@@ -53,47 +53,23 @@
         elif status == SEEN_CONSTANT_INT:
             if self.is_int(w_value):
                 if self.read_constant_int() != self.get_int_val(w_value):
-                    if not jit.we_are_jitted():
-                        if self._vprof_counter >= 200:
-                            print "NO LONGER CONSTANT", self._vprof_msg, 'int', w_value
                     self._vprof_status = SEEN_CONSTANT_CLASS
                     self._vprof_const_cls = w_value.__class__
-                else:
-                    if not jit.we_are_jitted():
-                        self._vprof_counter += 1
-                        if self._vprof_counter == 200:
-                            print self._vprof_msg, 'int', w_value
             else:
-                if self._vprof_counter >= 200:
-                    print "NO LONGER CONSTANT", self._vprof_msg, 'obj', w_value
                 self._vprof_status = SEEN_TOO_MUCH
         elif status == SEEN_CONSTANT_OBJ:
             prev_obj = self.try_read_constant_obj()
             if prev_obj is not w_value:
-                if self._vprof_counter >= 200:
-                    print "NO LONGER CONSTANT", self._vprof_msg, 'obj', w_value
                 prev_cls = self.read_constant_cls()
                 if prev_cls is w_value.__class__:
                     self._vprof_const_cls = prev_cls
                     self._vprof_status = SEEN_CONSTANT_CLASS
                 else:
                     self._vprof_status = SEEN_TOO_MUCH
-            else:
-                if not jit.we_are_jitted():
-                    self._vprof_counter += 1
-                    if self._vprof_counter == 200:
-                        print self._vprof_msg, 'obj', w_value
         elif status == SEEN_CONSTANT_CLASS:
             cls = self.read_constant_cls()
             if cls is not w_value.__class__:
                 self._vprof_status = SEEN_TOO_MUCH
-                if self._vprof_counter >= 200:
-                    print "NO LONGER CONSTANT CLASS", self._vprof_msg, 'cls', cls
-            else:
-                if not jit.we_are_jitted():
-                    self._vprof_counter += 1
-                    if self._vprof_counter == 200:
-                        print self._vprof_msg, 'cls', cls
 
     def can_fold_read_int(self):
         return self._vprof_status == SEEN_CONSTANT_INT

From noreply at buildbot.pypy.org  Tue Aug 18 12:07:48 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Tue, 18 Aug 2015 12:07:48 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: for local variables,
 only track the type to make the interpreter less bad
Message-ID: <20150818100748.1FB341C023F@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79032:4c1c11cfc6a9
Date: 2015-08-18 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/4c1c11cfc6a9/

Log:	for local variables, only track the type to make the interpreter
	less bad

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -50,22 +50,15 @@
     kwargname = varnames[argcount] if code.co_flags & CO_VARKEYWORDS else None
     return Signature(argnames, varargname, kwargname)
 
-class ValueProf(valueprof.ValueProf):
-    def is_int(self, w_obj):
-        from pypy.objspace.std.intobject import W_IntObject
-        return type(w_obj) is W_IntObject
-
-    def get_int_val(self, w_obj):
-        from pypy.objspace.std.intobject import W_IntObject
-        assert isinstance(w_obj, W_IntObject)
-        return w_obj.intval
+class KnownTypesVersion(object):
+    pass
 
 class PyCode(eval.Code):
     "CPython-style code objects."
     _immutable_ = True
     _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
                           "co_freevars[*]", "co_cellvars[*]",
-                          "_args_as_cellvars[*]", "vprofs[*]"]
+                          "_args_as_cellvars[*]", "_known_types_version?"]
 
     def __init__(self, space,  argcount, nlocals, stacksize, flags,
                      code, consts, names, varnames, filename,
@@ -95,7 +88,13 @@
         self._signature = cpython_code_signature(self)
         self._initialize()
         self._init_ready()
-        self.vprofs = [ValueProf('%s %s' % (self.co_name, self.co_varnames[i])) for i in range(self.co_nlocals)]
+        # a list of either None, W_Root, or a subclass thereof
+        # None means "have not seen a value in that local variable yet
+        # W_Root can be anything
+        # otherwise it's the precise class of *all* values ever stored in that
+        # local
+        self._known_types = [None] * self.co_nlocals
+        self._known_types_version = KnownTypesVersion()
 
     def _initialize(self):
         if self.co_cellvars:
@@ -140,6 +139,21 @@
     def _init_ready(self):
         "This is a hook for the vmprof module, which overrides this method."
 
+    def _get_known_type(self, varindex):
+        # somewhat subtle:
+        if not jit.we_are_jitted():
+            return self._known_types[varindex]
+        return self._get_known_type_elidable(varindex, self._known_types_version)
+
+    @jit.elidable
+    def _get_known_type_elidable(self, varindex, version):
+        assert version is self._known_types_version
+        return self._known_types[varindex]
+
+    def _update_known_type(self, varindex, cls):
+        self._known_types[varindex] = cls
+        self._known_types_version = KnownTypesVersion()
+
     def _cleanup_(self):
         if (self.magic == cpython_magic and
             '__pypy__' not in sys.builtin_module_names):
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -4,7 +4,7 @@
 from rpython.rlib import jit
 from rpython.rlib.debug import make_sure_not_resized, check_nonneg
 from rpython.rlib.jit import hint, we_are_jitted
-from rpython.rlib.objectmodel import we_are_translated, instantiate
+from rpython.rlib.objectmodel import we_are_translated, instantiate, specialize
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.tool.pairtype import extendabletype
 
@@ -146,36 +146,38 @@
         return cell
 
     def _getlocal(self, varindex):
-        from pypy.objspace.std.intobject import W_IntObject
-        # some careful logic there
-        if we_are_jitted():
-            vprof = self.getcode().vprofs[varindex]
-            if vprof.can_fold_read_int():
-                return W_IntObject(vprof.read_constant_int())
-            elif vprof.can_fold_read_obj():
-                w_res = vprof.try_read_constant_obj()
-                if w_res is not None:
-                    return w_res
         w_res = self.locals_cells_stack_w[varindex]
         if we_are_jitted():
-            vprof = self.getcode().vprofs[varindex]
-            if vprof.class_is_known():
-                jit.record_exact_class(w_res, vprof.read_constant_cls())
+            cls = self.getcode()._get_known_type(varindex)
+            if cls is not None and cls is not W_Root:
+                jit.record_exact_class(w_res, cls)
         return w_res
 
-    def _setlocal(self, varindex, value):
-        self._value_profile_local(varindex, value)
+    @specialize.arg(3)
+    def _setlocal(self, varindex, value, can_be_None=True):
+        self._see_write(varindex, value, can_be_None)
         self.locals_cells_stack_w[varindex] = value
 
-    def _value_profile_local(self, varindex, value):
-        from pypy.objspace.std.intobject import W_IntObject
-        vprof = self.getcode().vprofs[varindex]
-        vprof.see_write(value)
+    @specialize.arg(3)
+    def _see_write(self, varindex, value, can_be_None=True):
+        cls = self.getcode()._get_known_type(varindex)
+        if cls is W_Root:
+            return
+        if can_be_None and value is None:
+            new_cls = W_Root
+        else:
+            new_cls = value.__class__
+            if cls is not None:
+                if new_cls is not cls:
+                    new_cls = W_Root
+                else:
+                    return
+        self.getcode()._update_known_type(varindex, new_cls)
 
     @jit.unroll_safe
     def _all_locals_changed(self):
-        for i, vprof in enumerate(self.getcode().vprofs):
-            vprof.see_write(self.locals_cells_stack_w[i])
+        for i in range(self.getcode().co_nlocals):
+            self._see_write(i, self.locals_cells_stack_w[i])
 
     def mark_as_escaped(self):
         """
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -505,7 +505,7 @@
     def STORE_FAST(self, varindex, next_instr):
         w_newvalue = self.popvalue()
         assert w_newvalue is not None
-        self._setlocal(varindex, w_newvalue)
+        self._setlocal(varindex, w_newvalue, can_be_None=False)
 
     def getfreevarname(self, index):
         freevarnames = self.pycode.co_cellvars + self.pycode.co_freevars
diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -537,15 +537,29 @@
         sys.settrace(None)
         assert res == 10
 
-class TestValueProf(object):
-    def test_argument_is_constant(self):
+class TestLocalTypeProfiling(object):
+    def test_argument_is_constant_type(self):
+        from pypy.objspace.std.intobject import W_IntObject
+        from pypy.interpreter.baseobjspace import W_Root
         space = self.space
         w_f = space.appexec([], """():
             def f(x):
                 y = x + 1
             return f""")
+        v1 = w_f.code._known_types_version
+        assert w_f.code._known_types == [None] * 2
         space.call_function(w_f, space.wrap(1))
-        assert len(w_f.code.vprofs) == 2
-        assert w_f.code.vprofs[0].can_fold_read_int()
-        assert w_f.code.vprofs[1].can_fold_read_int()
+        v2 = w_f.code._known_types_version
+        assert v2 is not v1
+        assert w_f.code._known_types == [W_IntObject] * 2
 
+        space.call_function(w_f, space.wrap(2))
+        v3 = w_f.code._known_types_version
+        assert v3 is v2
+        assert w_f.code._known_types == [W_IntObject] * 2
+
+        space.call_function(w_f, space.wrap(1.1))
+        v4 = w_f.code._known_types_version
+        assert v4 is not v3
+        assert w_f.code._known_types == [W_Root] * 2
+

From noreply at buildbot.pypy.org  Tue Aug 18 12:22:01 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 12:22:01 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: changes to get translation going,
 vec_params is a str parameter like enabled_opts
Message-ID: <20150818102201.CD5481C023F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79033:b5a77dd8c0bf
Date: 2015-08-18 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/b5a77dd8c0bf/

Log:	changes to get translation going, vec_params is a str parameter like
	enabled_opts

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -579,7 +579,9 @@
               'vec': 0,
               'vec_params': '0:0:50:0.60',
               }
+STR_PARAMETERS = ['enable_opts','vec_params']
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
+string_parameters = unrolling_iterable(STR_PARAMETERS)
 
 # ____________________________________________________________
 
@@ -797,11 +799,11 @@
             raise ValueError
         name = parts[0]
         value = parts[1]
-        if name == 'enable_opts':
-            set_param(driver, 'enable_opts', value)
+        if name in string_parameters:
+            set_param(driver, name, value)
         else:
             for name1, _ in unroll_parameters:
-                if name1 == name and name1 != 'enable_opts':
+                if name1 == name and name1 not in string_parameters:
                     try:
                         set_param(driver, name1, int(value))
                     except ValueError:
@@ -965,7 +967,7 @@
     def compute_result_annotation(self, s_driver, s_name, s_value):
         from rpython.annotator import model as annmodel
         assert s_name.is_constant()
-        if s_name.const == 'enable_opts':
+        if s_name.const in string_parameters:
             assert annmodel.SomeString(can_be_None=True).contains(s_value)
         else:
             assert (s_value == annmodel.s_None or
@@ -980,7 +982,7 @@
         hop.exception_cannot_occur()
         driver = hop.inputarg(lltype.Void, arg=0)
         name = hop.args_s[1].const
-        if name == 'enable_opts':
+        if name in string_parameters:
             repr = string_repr
         else:
             repr = lltype.Signed

From noreply at buildbot.pypy.org  Tue Aug 18 12:53:39 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 18 Aug 2015 12:53:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fix translation.
Message-ID: <20150818105339.2F5AD1C123B@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79034:ff829278fa73
Date: 2015-08-18 12:51 +0200
http://bitbucket.org/pypy/pypy/changeset/ff829278fa73/

Log:	Fix translation.

diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py
--- a/pypy/module/time/interp_time.py
+++ b/pypy/module/time/interp_time.py
@@ -786,6 +786,8 @@
         return space.wrap((float(kernel_time2) + float(user_time2)) * 1e-7)
 
 else:
+    have_times = hasattr(rposix, 'c_times')
+
     def process_time(space):
         if cConfig.has_clock_gettime and (
                 cConfig.CLOCK_PROF is not None or
@@ -806,7 +808,7 @@
                 if ret == 0:
                     return space.wrap(decode_timeval(rusage.c_ru_utime) +
                                       decode_timeval(rusage.c_ru_stime))
-        if hasattr(rposix, 'c_times'):
+        if have_times:
             with lltype.scoped_alloc(rposix.TMS) as tms:
                 ret = rposix.c_times(tms)
                 if ret != -1:
diff --git a/rpython/rlib/rtime.py b/rpython/rlib/rtime.py
--- a/rpython/rlib/rtime.py
+++ b/rpython/rlib/rtime.py
@@ -167,7 +167,7 @@
     c_clock_gettime = external('clock_gettime',
                                [lltype.Signed, lltype.Ptr(TIMESPEC)],
                                rffi.INT, releasegil=False)
-else:
+if need_rusage:
     RUSAGE = RUSAGE
     RUSAGE_SELF = RUSAGE_SELF or 0
     c_getrusage = external('getrusage', 

From noreply at buildbot.pypy.org  Tue Aug 18 13:13:58 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 13:13:58 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: set_param second parameter needs
 to be specialized, thus cannot be variable
Message-ID: <20150818111358.46CA61C0695@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79035:f874a9f93f05
Date: 2015-08-18 12:48 +0200
http://bitbucket.org/pypy/pypy/changeset/f874a9f93f05/

Log:	set_param second parameter needs to be specialized, thus cannot be
	variable

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -579,7 +579,7 @@
               'vec': 0,
               'vec_params': '0:0:50:0.60',
               }
-STR_PARAMETERS = ['enable_opts','vec_params']
+STR_PARAMETERS = ('enable_opts','vec_params')
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 string_parameters = unrolling_iterable(STR_PARAMETERS)
 
@@ -799,11 +799,13 @@
             raise ValueError
         name = parts[0]
         value = parts[1]
-        if name in string_parameters:
-            set_param(driver, name, value)
+        if name == 'enable_opts':
+            set_param(driver, 'enable_opts', value)
+        elif name == 'vec_param':
+            set_param(driver, 'vec_params', value)
         else:
             for name1, _ in unroll_parameters:
-                if name1 == name and name1 not in string_parameters:
+                if name1 == name and name1 not in STR_PARAMETERS:
                     try:
                         set_param(driver, name1, int(value))
                     except ValueError:
@@ -967,7 +969,7 @@
     def compute_result_annotation(self, s_driver, s_name, s_value):
         from rpython.annotator import model as annmodel
         assert s_name.is_constant()
-        if s_name.const in string_parameters:
+        if s_name.const in STR_PARAMETERS:
             assert annmodel.SomeString(can_be_None=True).contains(s_value)
         else:
             assert (s_value == annmodel.s_None or
@@ -982,7 +984,7 @@
         hop.exception_cannot_occur()
         driver = hop.inputarg(lltype.Void, arg=0)
         name = hop.args_s[1].const
-        if name in string_parameters:
+        if name in STR_PARAMETERS:
             repr = string_repr
         else:
             repr = lltype.Signed

From noreply at buildbot.pypy.org  Tue Aug 18 13:14:00 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 13:14:00 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: vec_params in interp_jit.py must
 be a separate case to wrap it in a w_str
Message-ID: <20150818111400.57F981C0695@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79036:483562ca6a0f
Date: 2015-08-18 13:14 +0200
http://bitbucket.org/pypy/pypy/changeset/483562ca6a0f/

Log:	vec_params in interp_jit.py must be a separate case to wrap it in a
	w_str

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,6 +137,8 @@
     for key, w_value in kwds_w.items():
         if key == 'enable_opts':
             jit.set_param(None, 'enable_opts', space.str_w(w_value))
+        elif key == 'vec_params':
+            jit.set_param(None, 'vec_params', space.str_w(w_value))
         else:
             intval = space.int_w(w_value)
             for name, _ in unroll_parameters:

From noreply at buildbot.pypy.org  Tue Aug 18 13:47:21 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 13:47:21 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: removed recently added
 STR_PARAMETERS as it does not allow specialization
Message-ID: <20150818114721.A457F1C123B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79037:328fce203b85
Date: 2015-08-18 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/328fce203b85/

Log:	removed recently added STR_PARAMETERS as it does not allow
	specialization

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -142,7 +142,7 @@
         else:
             intval = space.int_w(w_value)
             for name, _ in unroll_parameters:
-                if name == key and name != 'enable_opts':
+                if name == key and name != 'enable_opts' and name != 'vec_params':
                     jit.set_param(None, name, intval)
                     break
             else:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -579,9 +579,7 @@
               'vec': 0,
               'vec_params': '0:0:50:0.60',
               }
-STR_PARAMETERS = ('enable_opts','vec_params')
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
-string_parameters = unrolling_iterable(STR_PARAMETERS)
 
 # ____________________________________________________________
 
@@ -805,7 +803,7 @@
             set_param(driver, 'vec_params', value)
         else:
             for name1, _ in unroll_parameters:
-                if name1 == name and name1 not in STR_PARAMETERS:
+                if name1 == name and name1 != 'vec_params' and name != 'enable_opts':
                     try:
                         set_param(driver, name1, int(value))
                     except ValueError:
@@ -984,7 +982,9 @@
         hop.exception_cannot_occur()
         driver = hop.inputarg(lltype.Void, arg=0)
         name = hop.args_s[1].const
-        if name in STR_PARAMETERS:
+        if name == 'enable_opts':
+            repr = string_repr
+        elif name == 'vec_params':
             repr = string_repr
         else:
             repr = lltype.Signed

From noreply at buildbot.pypy.org  Tue Aug 18 14:00:12 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 14:00:12 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: removed last usage of STR_PARAMS
Message-ID: <20150818120012.C0B151C150B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79038:13d3acac4d6a
Date: 2015-08-18 14:00 +0200
http://bitbucket.org/pypy/pypy/changeset/13d3acac4d6a/

Log:	removed last usage of STR_PARAMS

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -967,7 +967,9 @@
     def compute_result_annotation(self, s_driver, s_name, s_value):
         from rpython.annotator import model as annmodel
         assert s_name.is_constant()
-        if s_name.const in STR_PARAMETERS:
+        if s_name.const == 'enable_opts':
+            assert annmodel.SomeString(can_be_None=True).contains(s_value)
+        elif s_name.const == 'vec_params':
             assert annmodel.SomeString(can_be_None=True).contains(s_value)
         else:
             assert (s_value == annmodel.s_None or

From noreply at buildbot.pypy.org  Tue Aug 18 14:40:04 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 14:40:04 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: typo name should be name1,
 added test case that creates cycle in dependency
Message-ID: <20150818124004.0AC431C023F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79039:61d63941604d
Date: 2015-08-18 14:40 +0200
http://bitbucket.org/pypy/pypy/changeset/61d63941604d/

Log:	typo name should be name1, added test case that creates cycle in
	dependency

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -69,6 +69,19 @@
     def cut_off_at(self, index):
         self.path = self.path[:index]
 
+    def check_acyclic(self):
+        """NOT_RPYTHON"""
+        seen = set()
+        for segment in self.path:
+            if segment in seen:
+                print "path:"
+                for segment in self.path:
+                    print " ->", segment
+                print ""
+                assert 0, "segment %s was already seen. this makes the path cyclic!" % segment
+            else:
+                seen.add(segment)
+
     def clone(self):
         return Path(self.path[:])
 
@@ -107,6 +120,8 @@
     def edge_to(self, to, arg=None, failarg=False, label=None):
         if self is to:
             return
+        if self.getindex() > to.getindex():
+            import pdb; pdb.set_trace()
         dep = self.depends_on(to)
         if not dep:
             #if force or self.independent(idx_from, idx_to):
@@ -805,7 +820,7 @@
     def operation_{name}(self, op, node):
         descr = op.getdescr()
         idx_ref = self.get_or_create(op.getarg(1))
-        if descr.is_array_of_primitives():
+        if descr and descr.is_array_of_primitives():
             node.memory_ref = MemoryRef(op, idx_ref, {raw_access})
             self.memory_refs[node] = node.memory_ref
     """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -356,5 +356,19 @@
         self.assert_dependencies(ops, full_check=False)
         self.assert_dependent(2,12)
 
+    def test_cyclic(self):
+        pass 
+        trace = """
+        [p0, p1, p5, p6, p7, p9, p11, p12] # 0: 1,6
+        guard_early_exit() [] # 1: 2,6,7
+        p13 = getfield_gc(p9) # 2: 3,4,5,6
+        guard_nonnull(p13) [] # 3: 4,5,6
+        i14 = getfield_gc(p9) # 4: 5,6,7
+        p15 = getfield_gc(p13) # 5: 6
+        guard_class(p15, 140737326900656) [p1, p0, p9, i14, p15, p13, p5, p6, p7] # 6: 7
+        jump(p0,p1,p5,p6,p7,p9,p11,p12) # 7:
+        """
+        self.assert_dependencies(trace, full_check=True)
+
 class TestLLtype(BaseTestDependencyGraph, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -90,6 +90,7 @@
             guard.setfailargs([])
             guard.setdescr(compile.ResumeAtLoopHeaderDescr())
             loop.operations.insert(idx+1, guard)
+        self.show_dot_graph(DependencyGraph(opt.loop), "original_" + self.test_name)
         opt.analyse_index_calculations()
         if opt.dependency_graph is not None:
             self.show_dot_graph(opt.dependency_graph, "early_exit_" + self.test_name)
@@ -1383,5 +1384,116 @@
         opt = self.schedule(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
 
+    def test_1(self):
+        trace = """
+        [p0, p1, p5, p6, p7, p9, p11, p12]
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #34 FOR_ITER')
+        guard_early_exit(descr=) [p1, p0, p5, p6, p7, p9]
+        p13 = getfield_gc(p9, descr=)
+        guard_nonnull(p13, descr=) [p1, p0, p9, p13, p5, p6, p7]
+        i14 = getfield_gc(p9, descr=)
+        p15 = getfield_gc(p13, descr=)
+        guard_class(p15, 140737326900656, descr=) [p1, p0, p9, i14, p15, p13, p5, p6, p7]
+        p17 = getfield_gc(p13, descr=)
+        i18 = getfield_gc_pure(p17, descr=)
+        i20 = int_lt(i14, 0)
+        guard_false(i20, descr=) [p1, p0, p9, i14, i18, p5, p6, p7]
+        i21 = int_ge(i14, i18)
+        guard_false(i21, descr=) [p1, p0, p9, i14, p5, p6, p7]
+        i23 = int_add(i14, 1)
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #37 STORE_NAME')
+        p24 = getfield_gc(p5, descr=)
+        setfield_gc(p9, i23, descr=)
+        guard_value(p24, 25, descr=) [p1, p0, p24, p5, p6, p7, p9, i14]
+        p26 = getfield_gc(p24, descr=)
+        guard_value(p26, 27, descr=) [p1, p0, p26, p24, p5, p6, p7, p9, i14]
+        guard_not_invalidated(descr=) [p1, p0, p24, p5, p6, p7, p9, i14]
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #40 LOAD_NAME')
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #43 CALL_FUNCTION')
+        p28 = force_token()
+        enter_portal_frame(15, 8070450532247933488)
+        debug_merge_point(1, 1, ' #0 LOAD_CONST')
+        debug_merge_point(1, 1, ' #3 STORE_FAST')
+        debug_merge_point(1, 1, ' #6 SETUP_LOOP')
+        debug_merge_point(1, 1, ' #9 LOAD_FAST')
+        debug_merge_point(1, 1, ' #12 LOAD_CONST')
+        debug_merge_point(1, 1, ' #15 COMPARE_OP')
+        debug_merge_point(1, 1, ' #18 POP_JUMP_IF_FALSE')
+        debug_merge_point(1, 1, ' #21 LOAD_GLOBAL')
+        debug_merge_point(1, 1, ' #24 LOAD_FAST')
+        debug_merge_point(1, 1, ' #27 BINARY_SUBSCR')
+        p32 = getfield_gc(31, descr=)
+        setfield_gc(33, i14, descr=)
+        guard_class(p32, 140737326657856, descr=) [p1, p0, p11, p32, p5, p6, p7, p9, p28, p12, None]
+        p36 = getfield_gc(31, descr=)
+        i37 = getfield_gc(p36, descr=)
+        i39 = uint_ge(0, i37)
+        guard_false(i39, descr=) [p1, p0, p11, i37, p36, p5, p6, p7, p9, p28, p12, None]
+        p40 = getfield_gc(p36, descr=)
+        i41 = getarrayitem_gc(p40, 0, descr=intarraydescr)
+        debug_merge_point(1, 1, ' #28 LOAD_CONST')
+        debug_merge_point(1, 1, ' #31 BINARY_ADD')
+        i43 = int_add_ovf(i41, 1)
+        guard_no_overflow(descr=) [p1, p0, p11, i43, p5, p6, p7, p9, i41, p28, p12, None]
+        debug_merge_point(1, 1, ' #32 LOAD_GLOBAL')
+        debug_merge_point(1, 1, ' #35 LOAD_FAST')
+        debug_merge_point(1, 1, ' #38 STORE_SUBSCR')
+        debug_merge_point(1, 1, ' #39 LOAD_FAST')
+        debug_merge_point(1, 1, ' #42 LOAD_CONST')
+        debug_merge_point(1, 1, ' #45 INPLACE_ADD')
+        debug_merge_point(1, 1, ' #46 STORE_FAST')
+        debug_merge_point(1, 1, ' #49 JUMP_ABSOLUTE')
+        i45 = getfield_raw(140737351875840, descr=)
+        setarrayitem_gc(p40, 0, i43, descr=intarraydescr)
+        i48 = int_lt(i45, 0)
+        guard_false(i48, descr=) [p1, p0, p11, p5, p6, p7, p9, None, p28, p12, None]
+        debug_merge_point(1, 1, ' #9 LOAD_FAST')
+        p49 = force_token()
+        p51 = new_with_vtable(140737326477112)
+        p53 = new_array_clear(4, descr=)
+        p55 = new_with_vtable(100000)
+        p57 = new_with_vtable(140737326516800)
+        setfield_gc(p57, p28, descr=)
+        setfield_gc(p57, 0, descr=)
+        setfield_gc(p11, p57, descr=)
+        setfield_gc(p0, p49, descr=)
+        setfield_gc(p51, 0, descr=)
+        setfield_gc(p51, 0, descr=)
+        setfield_gc(p51, 9, descr=)
+        setfield_gc(p51, 1, descr=)
+        setfield_gc(p51, 25, descr=)
+        setfield_gc(p51, 63, descr=)
+        setfield_gc(p55, 1, descr=)
+        setarrayitem_gc(p53, 0, p55, descr=)
+        setfield_gc(p51, p53, descr=)
+        setfield_gc(p51, p12, descr=)
+        setfield_gc(p51, 66, descr=)
+        p67 = call_assembler(p51, p11, descr=)
+        guard_not_forced(descr=) [p1, p0, p11, p51, p67, p57, p5, p6, p7, p9]
+        keepalive(p51)
+        guard_no_exception(descr=) [p1, p0, p11, p51, p67, p57, p5, p6, p7, p9]
+        leave_portal_frame(15)
+        p69 = getfield_gc(p11, descr=)
+        p70 = getfield_gc(p51, descr=)
+        i71 = getfield_gc(p51, descr=)
+        setfield_gc(p11, p70, descr=)
+        guard_false(i71, descr=) [p1, p0, p67, p69, p51, p57, p5, p6, p7, p9]
+        i72 = getfield_gc(p5, descr=)
+        setfield_gc(p57, 0, descr=)
+        guard_value(i72, 0, descr=) [i72, p1, p0, p5, p6, p7, p9, p67]
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #46 POP_TOP')
+        p75 = getfield_gc(p5, descr=)
+        guard_isnull(p75, descr=) [p1, p0, p75, p5, p6, p7, p9, p67]
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #47 JUMP_ABSOLUTE')
+        guard_not_invalidated(descr=) [p1, p0, p5, p6, p7, p9]
+        i77 = getfield_raw(140737351875840, descr=)
+        i79 = int_lt(i77, 0)
+        guard_false(i79, descr=) [p1, p0, p5, p6, p7, p9]
+        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #34 FOR_ITER')
+        jump(p0, p1, p5, p6, p7, p9, p11, p70)
+        """        
+        opt = self.schedule(self.parse_loop(trace))
+        self.debug_print_operations(opt.loop)
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -521,6 +521,8 @@
             modify_later = []
             last_prev_node = None
             for path in guard_node.iterate_paths(ee_guard_node, True):
+                if not we_are_translated():
+                    path.check_acyclic()
                 prev_node = path.second()
                 dep = prev_node.depends_on(guard_node)
                 if dep.is_failarg():
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -179,7 +179,7 @@
         primitive type (int,float) """
         if self.is_primitive_load() or self.is_primitive_store():
             descr = self.getdescr()
-            if descr.is_array_of_primitives():
+            if descr and descr.is_array_of_primitives():
                 return True
         return False
 
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -803,7 +803,7 @@
             set_param(driver, 'vec_params', value)
         else:
             for name1, _ in unroll_parameters:
-                if name1 == name and name1 != 'vec_params' and name != 'enable_opts':
+                if name1 == name and name1 != 'vec_params' and name1 != 'enable_opts':
                     try:
                         set_param(driver, name1, int(value))
                     except ValueError:

From noreply at buildbot.pypy.org  Tue Aug 18 17:08:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 17:08:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: added two tests ensuring that
 paths are constructed correctly, Path was not copied
Message-ID: <20150818150826.4786F1C123B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79040:4c2ccf278516
Date: 2015-08-18 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/4c2ccf278516/

Log:	added two tests ensuring that paths are constructed correctly, Path
	was not copied ensured that paths are not cyclic

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -26,8 +26,10 @@
 
 LOAD_COMPLEX_OBJ = [ (rop.GETARRAYITEM_GC, 0, 1)
                    , (rop.GETARRAYITEM_RAW, 0, 1)
+                   , (rop.RAW_LOAD, 0, 1)
                    , (rop.GETINTERIORFIELD_GC, 0, 1)
-                   , (rop.RAW_LOAD, 0, 1)
+                   , (rop.GETFIELD_GC, 0, -1)
+                   , (rop.GETFIELD_RAW, 0, -1)
                    ]
 
 class Path(object):
@@ -44,7 +46,7 @@
             return None
         return self.path[len(self.path)-2]
 
-    def has_no_side_effects(self, exclude_first=False, exclude_last=False):
+    def is_always_pure(self, exclude_first=False, exclude_last=False):
         last = len(self.path)-1
         count = len(self.path)
         i = 0
@@ -81,10 +83,15 @@
                 assert 0, "segment %s was already seen. this makes the path cyclic!" % segment
             else:
                 seen.add(segment)
+        return True
 
     def clone(self):
         return Path(self.path[:])
 
+    def as_str(self):
+        """ NOT_RPYTHON """
+        return ' -> '.join([str(p) for p in self.path])
+
 class Node(object):
     def __init__(self, op, opidx):
         self.op = op
@@ -120,8 +127,6 @@
     def edge_to(self, to, arg=None, failarg=False, label=None):
         if self is to:
             return
-        if self.getindex() > to.getindex():
-            import pdb; pdb.set_trace()
         dep = self.depends_on(to)
         if not dep:
             #if force or self.independent(idx_from, idx_to):
@@ -161,7 +166,7 @@
         return self.op.getopnum() == rop.GUARD_EARLY_EXIT
 
     def loads_from_complex_object(self):
-        return rop._ALWAYS_PURE_LAST <= self.op.getopnum() <= rop.GETINTERIORFIELD_GC
+        return rop._ALWAYS_PURE_LAST <= self.op.getopnum() < rop._MALLOC_FIRST
 
     def modifies_complex_object(self):
         return rop.SETARRAYITEM_GC <= self.op.getopnum() <= rop.UNICODESETITEM
@@ -183,18 +188,17 @@
                         args.append((op.getarg(i), op.getarg(j), True))
                         for x in range(j+1,len(op_args)):
                             args.append((op.getarg(x), None, False))
-                    break
-        else:
-            # assume this destroys every argument... can be enhanced by looking
-            # at the effect info of a call for instance
-            for arg in op.getarglist():
-                # if it is a constant argument it cannot be destroyed.
-                # neither can a box float be destroyed. BoxInt can
-                # contain a reference thus it is assumed to be destroyed
-                if isinstance(arg, Const) or isinstance(arg, BoxFloat):
-                    args.append((arg, None, False))
-                else:
-                    args.append((arg, None,True))
+                    return args
+        # assume this destroys every argument... can be enhanced by looking
+        # at the effect info of a call for instance
+        for arg in op.getarglist():
+            # if it is a constant argument it cannot be destroyed.
+            # neither can a box float be destroyed. BoxInt can
+            # contain a reference thus it is assumed to be destroyed
+            if isinstance(arg, Const) or isinstance(arg, BoxFloat):
+                args.append((arg, None, False))
+            else:
+                args.append((arg, None, True))
         return args
 
     def provides_count(self):
@@ -259,7 +263,7 @@
 
     def iterate_paths(self, to, backwards=False, path_max_len=-1):
         """ yield all nodes from self leading to 'to' """
-        if self == to:
+        if self is to:
             return
         path = Path([self])
         worklist = [(0, self, 1)]
@@ -282,7 +286,7 @@
                 pathlen += 1
 
                 if next_node is to or (path_max_len > 0 and pathlen >= path_max_len):
-                    yield path
+                    yield Path(path.path[:])
                 else:
                     worklist.append((0, next_node, pathlen))
 
@@ -586,7 +590,6 @@
                 # i = int_and(j, 255)
                 # guard_true(i) [...]
                 pass
-
         elif guard_op.is_foldable_guard():
             # these guards carry their protected variables directly as a parameter
             for arg in guard_node.getoperation().getarglist():
@@ -669,9 +672,12 @@
             for opnum, i, j in unrolling_iterable(LOAD_COMPLEX_OBJ):
                 if opnum == op.getopnum():
                     cobj = op.getarg(i)
-                    index_var = op.getarg(j)
-                    tracker.depends_on_arg(cobj, node, index_var)
-                    tracker.depends_on_arg(index_var, node)
+                    if j != -1:
+                        index_var = op.getarg(j)
+                        tracker.depends_on_arg(cobj, node, index_var)
+                        tracker.depends_on_arg(index_var, node)
+                    else:
+                        tracker.depends_on_arg(cobj, node)
         else:
             for arg, argcell, destroyed in node.side_effect_arguments():
                 if argcell is not None:
@@ -722,6 +728,8 @@
             dot = "digraph dep_graph {\n"
             for node in self.nodes:
                 op = node.getoperation()
+                if op.getopnum() == rop.DEBUG_MERGE_POINT:
+                    continue
                 op_str = str(op)
                 if op.is_guard():
                     op_str += " " + ','.join([str(arg) for arg in op.getfailargs()])
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -5,10 +5,18 @@
     LLtypeMixin, BaseTest, FakeMetaInterpStaticData, convert_old_style_to_targets)
 from rpython.jit.metainterp.history import TargetToken, JitCellToken, TreeLoop
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph, Dependency,
-        IndexVar, MemoryRef)
+        IndexVar, MemoryRef, Node)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.conftest import option
 
+class FakeNode(Node):
+    def __init__(self, i):
+        Node.__init__(self, None, i)
+        pass
+
+    def __repr__(self):
+        return "n%d" % self.opidx
+
 class DependencyBaseTest(BaseTest):
 
     def setup_method(self, method):
@@ -356,19 +364,75 @@
         self.assert_dependencies(ops, full_check=False)
         self.assert_dependent(2,12)
 
+    def test_getfield(self):
+        pass 
+        trace = """
+        [p0, p1] # 0: 1,2,5
+        p2 = getfield_gc(p0) # 1: 3,5
+        p3 = getfield_gc(p0) # 2: 4
+        guard_nonnull(p2) [p2] # 3: 4,5
+        guard_nonnull(p3) [p3] # 4: 5
+        jump(p0,p2) # 5:
+        """
+        self.assert_dependencies(trace, full_check=True)
+
     def test_cyclic(self):
         pass 
         trace = """
         [p0, p1, p5, p6, p7, p9, p11, p12] # 0: 1,6
-        guard_early_exit() [] # 1: 2,6,7
-        p13 = getfield_gc(p9) # 2: 3,4,5,6
-        guard_nonnull(p13) [] # 3: 4,5,6
-        i14 = getfield_gc(p9) # 4: 5,6,7
+        guard_early_exit() [] # 1: 2,4,6,7
+        p13 = getfield_gc(p9) # 2: 3,5,6
+        guard_nonnull(p13) [] # 3: 5,6
+        i14 = getfield_gc(p9) # 4: 6
         p15 = getfield_gc(p13) # 5: 6
         guard_class(p15, 140737326900656) [p1, p0, p9, i14, p15, p13, p5, p6, p7] # 6: 7
         jump(p0,p1,p5,p6,p7,p9,p11,p12) # 7:
         """
         self.assert_dependencies(trace, full_check=True)
 
+
+    def test_iterate_paths(self):
+        n1,n2,n3,n4,n5 = [FakeNode(i+1) for i in range(5)]
+        # n1 -> n2 -> n4 -> n5
+        #  +---> n3 --^
+        n1.edge_to(n2); n2.edge_to(n4); n4.edge_to(n5)
+        n1.edge_to(n3); n3.edge_to(n4);
+
+        paths = list(n5.iterate_paths(n1, backwards=True))
+        assert all([path.check_acyclic() for path in paths])
+        assert len(paths) == 2
+        assert paths[0].as_str() == "n5 -> n4 -> n2 -> n1"
+        assert paths[1].as_str() == "n5 -> n4 -> n3 -> n1"
+        paths = list(n1.iterate_paths(n5))
+        assert all([path.check_acyclic() for path in paths])
+        assert len(paths) == 2
+        assert paths[0].as_str() == "n1 -> n2 -> n4 -> n5"
+        assert paths[1].as_str() == "n1 -> n3 -> n4 -> n5"
+
+
+    def test_iterate_one_many_one(self):
+        r = range(19)
+        n0 = FakeNode(0)
+        nodes = [FakeNode(i+1) for i in r]
+        nend = FakeNode(len(r)+1)
+
+        assert len(list(n0.iterate_paths(nodes[0], backwards=True))) == 0
+
+        for i in r:
+            n0.edge_to(nodes[i])
+            nodes[i].edge_to(nend)
+
+        paths = list(nend.iterate_paths(n0, backwards=True))
+        assert all([path.check_acyclic() for path in paths])
+        assert len(paths) == len(r)
+        for i in r:
+            assert paths[i].as_str() == "n%d -> %s -> n0" % (len(r)+1, nodes[i])
+        # forward
+        paths = list(n0.iterate_paths(nend))
+        assert all([path.check_acyclic() for path in paths])
+        assert len(paths) == len(r)
+        for i in r:
+            assert paths[i].as_str() == "n0 -> %s -> n%d" % (nodes[i], len(r)+1)
+
 class TestLLtype(BaseTestDependencyGraph, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -520,9 +520,12 @@
                 continue
             modify_later = []
             last_prev_node = None
+            i = 0
             for path in guard_node.iterate_paths(ee_guard_node, True):
                 if not we_are_translated():
                     path.check_acyclic()
+                print "loop", i
+                i+=1
                 prev_node = path.second()
                 dep = prev_node.depends_on(guard_node)
                 if dep.is_failarg():
@@ -540,7 +543,7 @@
                         continue
                     modify_later.append((prev_node, guard_node))
                 else:
-                    if path.has_no_side_effects(exclude_first=True, exclude_last=True):
+                    if path.is_always_pure(exclude_first=True, exclude_last=True):
                         path.set_schedule_priority(10)
                         modify_later.append((path.last_but_one(), None))
                     else:

From noreply at buildbot.pypy.org  Tue Aug 18 17:22:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 18 Aug 2015 17:22:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fight, the best I got so far
Message-ID: <20150818152226.5D67D1C023F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79041:f10f772c7fb6
Date: 2015-08-18 17:22 +0200
http://bitbucket.org/pypy/pypy/changeset/f10f772c7fb6/

Log:	fight, the best I got so far

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -244,11 +244,6 @@
         # mapping const value -> info corresponding to it's heap cache
         self.const_infos = self.optimizer.cpu.ts.new_ref_dict()
 
-    def force_at_end_of_preamble(self):
-        self.cached_dict_reads.clear()
-        self.corresponding_array_descrs.clear()
-        self.force_all_lazy_setfields_and_arrayitems()
-
     def flush(self):
         self.cached_dict_reads.clear()
         self.corresponding_array_descrs.clear()
@@ -478,6 +473,11 @@
             if indexb is None or indexb.contains(idx):
                 cf.force_lazy_setfield(self, None, can_cache)
 
+    def force_at_end_of_preamble(self):
+        self.cached_dict_reads.clear()
+        self.corresponding_array_descrs.clear()
+        self.force_all_lazy_setfields_and_arrayitems()
+
     def force_all_lazy_setfields_and_arrayitems(self):
         items = self.cached_fields.items()
         if not we_are_translated():
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -137,7 +137,7 @@
             v1, v2 = v2, v1
         # if both are constant, the pure optimization will deal with it
         if v2.is_constant() and not v1.is_constant():
-            if False and not self.optimizer.is_inputarg(arg1):
+            if not self.optimizer.is_inputarg(arg1):
                 if arg1.getopnum() == rop.INT_ADD:
                     prod_arg1 = arg1.getarg(0)
                     prod_arg2 = arg1.getarg(1)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -41,6 +41,7 @@
 class AbstractShortOp(object):
     """ An operation that is potentially produced by the short preamble
     """
+    pass
 
 class HeapOp(AbstractShortOp):
     def __init__(self, res, getfield_op):
@@ -209,15 +210,15 @@
         # of AbstractShortOp
         self.potential_ops = OrderedDict()
         self.produced_short_boxes = {}
-        self.short_inputargs = []
         # a way to produce const boxes, e.g. setfield_gc(p0, Const).
         # We need to remember those, but they don't produce any new boxes
         self.const_short_boxes = []
-        for i in range(len(inputargs)):
-            box = inputargs[i]
+        self.short_inputargs = []
+        for i in range(len(label_args)):
+            box = label_args[i]
             renamed = OpHelpers.inputarg_from_tp(box.type)
-            self.potential_ops[box] = ShortInputArg(label_args[i], renamed)
             self.short_inputargs.append(renamed)
+            self.potential_ops[box] = ShortInputArg(box, renamed)
 
         optimizer.produce_potential_short_preamble_ops(self)
 
@@ -302,7 +303,7 @@
                 renamed = OpHelpers.inputarg_from_tp(label_args[i].type)
                 short_inpargs.append(renamed)
             else:
-                #assert isinstance(inparg, ShortInputArg)
+                assert isinstance(inparg.short_op, ShortInputArg)
                 short_inpargs.append(inparg.preamble_op)
         return short_inpargs
 
@@ -314,7 +315,8 @@
         self.potential_ops[op] = CompoundOp(op, pop, prev_op)
 
     def add_pure_op(self, op):
-        assert op not in self.potential_ops
+        if op in self.potential_ops:
+            return
         self.add_potential_op(op, PureOp(op))
 
     def add_loopinvariant_op(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -879,7 +879,7 @@
         jump(p2, p3)
         """
         short = """
-        [p1]
+        [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
         jump(i1)
         """
@@ -1221,12 +1221,10 @@
         setfield_gc(p0, p1, descr=nextdescr)
         setfield_gc(p2, i1, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1)
+        jump(p1, p2, i1)
         """
         loop = """
-        [p0]
-        p41 = getfield_gc_r(p0, descr=nextdescr)
-        i0 = getfield_gc_i(p41, descr=valuedescr)
+        [p0, p41, i0]
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(descr=nodesize2)
         p2 = new_with_vtable(descr=nodesize2)
@@ -1234,7 +1232,7 @@
         setfield_gc(p0, p1, descr=nextdescr)
         setfield_gc(p2, i1, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1)
+        jump(p1, p2, i1)
         """
         self.optimize_loop(ops, loop, preamble)
 
@@ -2397,7 +2395,11 @@
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2, 1) # , i5)
         """
-        self.optimize_loop(ops, expected, preamble)
+        expected_short = """
+        []
+        """
+        self.optimize_loop(ops, expected, preamble,
+                           expected_short=expected_short)
 
     def test_duplicate_setfield_residual_guard_2(self):
         # the difference with the previous test is that the field value is
@@ -8718,5 +8720,16 @@
         """
         self.optimize_loop(ops, expected, expected_short=expected_short)
 
+    def test_simple_swap_arguments(self):
+        ops = """
+        [i0, i1]
+        i2 = int_add(i1, 1)
+        jump(i2, i0)
+        """
+        expected = """
+        []
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_unroll.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      InputArgRef
 from rpython.jit.metainterp.optimizeopt.shortpreamble import \
-     ShortPreambleBuilder, PreambleOp
+     ShortPreambleBuilder, PreambleOp, ShortInputArg
 from rpython.jit.metainterp.compile import LoopCompileData
 from rpython.jit.metainterp.optimizeopt.virtualstate import \
      NotVirtualStateInfo, LEVEL_CONSTANT, LEVEL_UNKNOWN, LEVEL_KNOWNCLASS,\
@@ -64,6 +64,9 @@
             remap[v] = k
         equaloplists(short, exp, remap=remap)
 
+def producable_short_boxes(l):
+    return [x for x in l if not isinstance(x.short_op, ShortInputArg)]
+        
 class TestUnroll(BaseTestUnroll):
     def test_simple(self):
         loop = """
@@ -230,7 +233,8 @@
         jump(i1, p1, p2)
         """
         es, loop, preamble = self.optimize(loop)
-        assert len(es.short_boxes) == 2
+        assert len(producable_short_boxes(es.short_boxes)) == 2
+        xxx
 
     def test_setfield_forced_virtual(self):
         loop = """
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -107,7 +107,10 @@
                      for op in end_jump.getarglist()]
         jump_args = state.virtual_state.make_inputargs(orig_jump_args,
                                     self.optimizer, force_boxes=True)
-        extra_jump_args = self.inline_short_preamble(jump_args)
+        pass_to_short = state.virtual_state.make_inputargs(orig_jump_args,
+                                    self.optimizer, force_boxes=True,
+                                    append_virtuals=True)
+        extra_jump_args = self.inline_short_preamble(pass_to_short)
         # remove duplicates, removes stuff from used boxes too
         label_args, jump_args = self.filter_extra_jump_args(
             start_label.getarglist() + self.short_preamble_producer.used_boxes,
@@ -166,9 +169,11 @@
         for arg in label_args:
             infos[arg] = self.optimizer.getinfo(arg)            
         sb = ShortBoxes()
+        label_args_plus_virtuals = virtual_state.make_inputargs(end_args,
+                                        self.optimizer, append_virtuals=True)
         short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs,
-                                            label_args)
-        short_inputargs = sb.create_short_inputargs(label_args)
+                                            label_args_plus_virtuals)
+        short_inputargs = sb.create_short_inputargs(label_args_plus_virtuals)
         for produced_op in short_boxes:
             op = produced_op.short_op.res
             if not isinstance(op, Const):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -516,7 +516,8 @@
                                           state)
         return state
 
-    def make_inputargs(self, inputargs, optimizer, force_boxes=False):
+    def make_inputargs(self, inputargs, optimizer, force_boxes=False,
+                       append_virtuals=False):
         assert len(inputargs) == len(self.state)
         boxes = [None] * self.numnotvirtuals
 
@@ -530,6 +531,14 @@
         for i in range(len(inputargs)):
             self.state[i].enum_forced_boxes(boxes, inputargs[i], optimizer)
 
+        if append_virtuals:
+            # we append the virtuals here in case some stuff is proven
+            # to be not a virtual and there are getfields in the short preamble
+            # that will read items out of there
+            for i in range(len(inputargs)):
+                if not isinstance(self.state[i], NotVirtualStateInfo):
+                    boxes.append(inputargs[i])
+            
         return boxes
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):

From noreply at buildbot.pypy.org  Tue Aug 18 17:49:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Tue, 18 Aug 2015 17:49:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: path iteration on a dependecy
 graph can enable blacklisting nodes that have already been visited.this
 cuts down the search space. this is only valid if a property must hold
 along all the paths and once for one path proven,
 any subpath must not be checked anymore (is the case in guard early exit
Message-ID: <20150818154926.9E0FF1C150B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79042:4886c173125a
Date: 2015-08-18 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/4886c173125a/

Log:	path iteration on a dependecy graph can enable blacklisting nodes
	that have already been visited.this cuts down the search space. this
	is only valid if a property must hold along all the paths and once
	for one path proven, any subpath must not be checked anymore (is the
	case in guard early exit

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -44,7 +44,12 @@
     def last_but_one(self):
         if len(self.path) < 2:
             return None
-        return self.path[len(self.path)-2]
+        return self.path[-2]
+
+    def last(self):
+        if len(self.path) < 1:
+            return None
+        return self.path[-1]
 
     def is_always_pure(self, exclude_first=False, exclude_last=False):
         last = len(self.path)-1
@@ -261,10 +266,16 @@
                 worklist.append(dep.to)
         return True
 
-    def iterate_paths(self, to, backwards=False, path_max_len=-1):
-        """ yield all nodes from self leading to 'to' """
+    def iterate_paths(self, to, backwards=False, path_max_len=-1, blacklist=False):
+        """ yield all nodes from self leading to 'to'. backwards determines
+        the iteration direction and blacklist marks nodes that have already been visited.
+        blacklist comes in handy if a property must hold for every path. not *every* possible
+        instance must be iterated, but trees that have already been visited can be ignored
+        after the have been visited
+        """
         if self is to:
             return
+        blacklist_visit = {}
         path = Path([self])
         worklist = [(0, self, 1)]
         while len(worklist) > 0:
@@ -274,19 +285,28 @@
             else:
                 iterdir = node.provides()
             if index >= len(iterdir):
+                if blacklist:
+                    blacklist_visit[node] = None
                 continue
             else:
                 next_dep = iterdir[index]
                 next_node = next_dep.to
                 index += 1
+                if blacklist and next_node in blacklist_visit:
+                    yield Path(path.path[:])
+                    continue
                 if index < len(iterdir):
                     worklist.append((index, node, pathlen))
+                else:
+                    blacklist_visit[node] = None
                 path.cut_off_at(pathlen)
                 path.walk(next_node)
                 pathlen += 1
 
                 if next_node is to or (path_max_len > 0 and pathlen >= path_max_len):
                     yield Path(path.path[:])
+                    if blacklist:
+                        blacklist_visit[next_node] = None
                 else:
                     worklist.append((0, next_node, pathlen))
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -434,5 +434,35 @@
         for i in r:
             assert paths[i].as_str() == "n0 -> %s -> n%d" % (nodes[i], len(r)+1)
 
+    def test_iterate_paths_blacklist_diamond(self):
+        blacklist = {}
+        n1,n2,n3,n4 = [FakeNode(i+1) for i in range(4)]
+        # n1 -> n2 -> n4
+        #  +---> n3 --^
+        n1.edge_to(n2); n2.edge_to(n4);
+        n1.edge_to(n3); n3.edge_to(n4);
+
+        paths = list(n1.iterate_paths(n4, blacklist=True))
+        assert len(paths) == 2
+        assert paths[0].as_str() == "n1 -> n2 -> n4"
+        assert paths[1].as_str() == "n1 -> n3"
+
+    def test_iterate_paths_blacklist_double_diamond(self):
+        blacklist = {}
+        n1,n2,n3,n4,n5,n6,n7,n8 = [FakeNode(i+1) for i in range(8)]
+        # n1 -> n2 -> n4 -> n5 -> n6 --> n8
+        #  +---> n3 --^      +---> n7 --^
+        n1.edge_to(n2); n2.edge_to(n4);
+        n1.edge_to(n3); n3.edge_to(n4);
+        n4.edge_to(n5)
+        n5.edge_to(n6); n6.edge_to(n8);
+        n5.edge_to(n7); n7.edge_to(n8);
+
+        paths = list(n1.iterate_paths(n8, blacklist=True))
+        assert len(paths) == 3
+        assert paths[0].as_str() == "n1 -> n2 -> n4 -> n5 -> n6 -> n8"
+        assert paths[1].as_str() == "n1 -> n2 -> n4 -> n5 -> n7"
+        assert paths[2].as_str() == "n1 -> n3"
+
 class TestLLtype(BaseTestDependencyGraph, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -521,7 +521,7 @@
             modify_later = []
             last_prev_node = None
             i = 0
-            for path in guard_node.iterate_paths(ee_guard_node, True):
+            for path in guard_node.iterate_paths(ee_guard_node, backwards=True, blacklist=True):
                 if not we_are_translated():
                     path.check_acyclic()
                 print "loop", i
@@ -545,7 +545,8 @@
                 else:
                     if path.is_always_pure(exclude_first=True, exclude_last=True):
                         path.set_schedule_priority(10)
-                        modify_later.append((path.last_but_one(), None))
+                        if path.last() is ee_guard_node:
+                            modify_later.append((path.last_but_one(), None))
                     else:
                         # transformation is invalid.
                         # exit and do not enter else branch!

From noreply at buildbot.pypy.org  Tue Aug 18 18:00:34 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 18 Aug 2015 18:00:34 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: I *think* this is correct
Message-ID: <20150818160034.8464E1C1C61@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79043:b2a9f450a0b4
Date: 2015-08-18 18:00 +0200
http://bitbucket.org/pypy/pypy/changeset/b2a9f450a0b4/

Log:	I *think* this is correct

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -196,7 +196,8 @@
         return ProducedShortOp(self, self.preamble_op)
 
     def produce_op(self, opt, preamble_op, exported_infos, invented_name):
-        assert not invented_name
+        pass
+        #assert not invented_name
 
     def __repr__(self):
         return "INP(%r -> %r)" % (self.res, self.preamble_op)
@@ -258,6 +259,19 @@
         else:
             return None
 
+    def _pick_op_index(self, lst, pick_inparg=True):
+        index = -1
+        for i, item in enumerate(lst):
+            if (not isinstance(item.short_op, HeapOp) and
+                (pick_inparg or not isinstance(item.short_op, ShortInputArg))):
+                if index != -1:
+                    assert pick_inparg
+                    return self._pick_op_index(lst, False)
+                index = i
+        if index == -1:
+            index = 0
+        return index
+
     def add_op_to_short(self, shortop):
         if shortop.res in self.produced_short_boxes:
             return # already added due to dependencies
@@ -268,13 +282,7 @@
                 if len(lst) == 0:
                     return None
                 else:
-                    index = -1
-                    for i, item in enumerate(lst):
-                        if not isinstance(item.short_op, HeapOp):
-                            assert index == -1
-                            index = i
-                    if index == -1:
-                        index = 0
+                    index = self._pick_op_index(lst)
                     pop = lst[index]
                     for i in range(len(lst)):
                         if i == index:
@@ -315,8 +323,6 @@
         self.potential_ops[op] = CompoundOp(op, pop, prev_op)
 
     def add_pure_op(self, op):
-        if op in self.potential_ops:
-            return
         self.add_potential_op(op, PureOp(op))
 
     def add_loopinvariant_op(self, op):
@@ -383,7 +389,7 @@
     def add_preamble_op(self, preamble_op):
         if preamble_op.invented_name:
             self.extra_same_as.append(preamble_op.op)
-        self.used_boxes.append(preamble_op.op)
+        self.used_boxes.append(preamble_op.op)            
         self.short_preamble_jump.append(preamble_op.preamble_op)
 
     def build_short_preamble(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -2395,11 +2395,7 @@
         setfield_gc(p1, i2, descr=valuedescr)
         jump(p1, i1, i2, 1) # , i5)
         """
-        expected_short = """
-        []
-        """
-        self.optimize_loop(ops, expected, preamble,
-                           expected_short=expected_short)
+        self.optimize_loop(ops, expected, preamble)
 
     def test_duplicate_setfield_residual_guard_2(self):
         # the difference with the previous test is that the field value is
@@ -8720,16 +8716,5 @@
         """
         self.optimize_loop(ops, expected, expected_short=expected_short)
 
-    def test_simple_swap_arguments(self):
-        ops = """
-        [i0, i1]
-        i2 = int_add(i1, 1)
-        jump(i2, i0)
-        """
-        expected = """
-        []
-        """
-        self.optimize_loop(ops, expected)
-
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Tue Aug 18 18:11:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 18 Aug 2015 18:11:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150818161148.2272C1C123B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79044:b3c4f481e9c3
Date: 2015-08-18 18:11 +0200
http://bitbucket.org/pypy/pypy/changeset/b3c4f481e9c3/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -5934,17 +5934,6 @@
         self.optimize_loop(ops, expected)
 
     def test_value_proven_to_be_constant_after_two_iterations(self):
-        class FakeDescr(AbstractDescr):
-            def __init__(self, name):
-                self.name = name
-            def sort_key(self):
-                return id(self)
-            def is_integer_bounded(self):
-                return False
-
-        for n in ('inst_w_seq', 'inst_index', 'inst_w_list', 'inst_length',
-                  'inst_start', 'inst_step'):
-            self.namespace[n] = FakeDescr(n)
         ops = """
         [p0, p1, p2, p3, i4, p5, i6, p7, p8, p9, p14]
         guard_value(i4, 3) []
@@ -6082,6 +6071,8 @@
             _TYPE = llmemory.GCREF.TO
             def __eq__(self, other):
                 return other.container.intval == 1242
+            def _normalizedcontainer(self):
+                return self
         self.namespace['intobj1242'] = lltype._ptr(llmemory.GCREF,
                                                    IntObj1242())
         expected = """
@@ -7233,7 +7224,6 @@
         p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
-        call_n(123, p25, descr=nonwritedescr)
         jump(p25, p187, i184)
         """
         preamble = """
@@ -7241,7 +7231,6 @@
         p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
-        call_n(123, p25, descr=nonwritedescr)
         jump(p25, p187, i184)
         """
         short = """
@@ -7252,9 +7241,8 @@
         jump(p25)
         """
         expected = """
-        [p25, p187, i184]
-        call_n(123, p25, descr=nonwritedescr)
-        jump(p25, p187, i184)
+        [p25, p187, i184, p189]
+        jump(p25, p187, i184, p189)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
@@ -8134,7 +8122,7 @@
         i4 = int_add(i3, i3)
         i5 = int_add(i4, i4)
         i6 = int_add(i5, i5)
-        #jump(i1, i2, i6, i3)
+        jump(i6, i3)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -187,6 +187,19 @@
     T = lltype.GcStruct('TUPLE',
                         ('c', lltype.Signed),
                         ('d', lltype.Ptr(lltype.GcArray(lltype.Ptr(NODE)))))
+
+    W_ROOT = lltype.GcStruct('W_ROOT', ('parent', OBJECT),
+        ('inst_w_seq', llmemory.GCREF), ('inst_index', lltype.Signed),
+        ('inst_w_list', llmemory.GCREF), ('inst_length', lltype.Signed),
+        ('inst_start', lltype.Signed), ('inst_step', lltype.Signed))
+    inst_w_seq = cpu.fielddescrof(W_ROOT, 'inst_w_seq')
+    inst_index = cpu.fielddescrof(W_ROOT, 'inst_index')
+    inst_length = cpu.fielddescrof(W_ROOT, 'inst_length')
+    inst_start = cpu.fielddescrof(W_ROOT, 'inst_start')
+    inst_step = cpu.fielddescrof(W_ROOT, 'inst_step')
+    inst_w_list = cpu.fielddescrof(W_ROOT, 'inst_w_list')
+    w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
+    
     tsize = cpu.sizeof(T, False)
     cdescr = cpu.fielddescrof(T, 'c')
     ddescr = cpu.fielddescrof(T, 'd')
@@ -339,6 +352,7 @@
     register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
     register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
     register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
+    register_known_gctype(cpu, w_root_vtable, W_ROOT)
 
     namespace = locals()
 

From noreply at buildbot.pypy.org  Tue Aug 18 19:30:46 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Aug 2015 19:30:46 +0200 (CEST)
Subject: [pypy-commit] pypy default: Move the _flush_codes logic to C
Message-ID: <20150818173046.28BE21C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79045:a5241c131e2a
Date: 2015-08-18 19:30 +0200
http://bitbucket.org/pypy/pypy/changeset/a5241c131e2a/

Log:	Move the _flush_codes logic to C

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -49,9 +49,10 @@
     vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
                                      compilation_info=eci,
                                      save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_write_buf = rffi.llexternal("vmprof_write_buf",
-                                       [rffi.CCHARP, rffi.LONG],
-                                       lltype.Void, compilation_info=eci)
+    vmprof_register_virtual_function = rffi.llexternal(
+                                           "vmprof_register_virtual_function",
+                                           [rffi.CCHARP, rffi.LONG, rffi.INT],
+                                           rffi.INT, compilation_info=eci)
     vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals",
                                             [rffi.INT], lltype.Void,
                                             compilation_info=eci)
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,14 +1,12 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import jit, rgc, rposix
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
 from rpython.rtyper.lltypesystem import rffi
 
-MAX_CODES = 8000 - 255
-MAX_FUNC_NAME = 255
+MAX_FUNC_NAME = 1023
 
 # ____________________________________________________________
 
@@ -34,8 +32,6 @@
 
     def _cleanup_(self):
         self.is_enabled = False
-        self.fileno = -1
-        self._current_codes = None
 
     @specialize.argtype(1)
     def register_code(self, code, full_name_func):
@@ -107,7 +103,6 @@
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
-        self.fileno = fileno
         self._gather_all_code_objs()
         res = self.cintf.vmprof_enable()
         if res < 0:
@@ -121,9 +116,6 @@
         if not self.is_enabled:
             raise VMProfError("vmprof is not enabled")
         self.is_enabled = False
-        if self._current_codes is not None:
-            self._flush_codes()
-        self.fileno = -1
         res = self.cintf.vmprof_disable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
@@ -132,35 +124,8 @@
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
             "and at most %d characters; got '%s'" % (MAX_FUNC_NAME, name))
-        b = self._current_codes
-        if b is None:
-            b = self._current_codes = StringBuilder()
-        b.append('\x02')
-        _write_long_to_string_builder(uid, b)
-        _write_long_to_string_builder(len(name), b)
-        b.append(name)
-        if b.getlength() >= MAX_CODES:
-            self._flush_codes()
-
-    def _flush_codes(self):
-        buf = self._current_codes.build()
-        self._current_codes = None
-        self.cintf.vmprof_write_buf(buf, len(buf))
-        # NOTE: keep in mind that vmprof_write_buf() can only write
-        # a maximum of 8184 bytes.  This should be guaranteed here because:
-        assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184
-
-
-def _write_long_to_string_builder(l, b):
-    b.append(chr(l & 0xff))
-    b.append(chr((l >> 8) & 0xff))
-    b.append(chr((l >> 16) & 0xff))
-    b.append(chr((l >> 24) & 0xff))
-    if sys.maxint > 2147483647:
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
+        if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
+            raise VMProfError("vmprof buffers full!  disk full or too slow")
 
 
 def vmprof_execute_code(name, get_code_fn, result_class=None):
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
 
-RPY_EXTERN char *vmprof_init(int, double, const char *);
+RPY_EXTERN char *vmprof_init(int, double, char *);
 RPY_EXTERN void vmprof_ignore_signals(int);
 RPY_EXTERN int vmprof_enable(void);
 RPY_EXTERN int vmprof_disable(void);
-RPY_EXTERN void vmprof_write_buf(char *, long);
+RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -45,11 +45,13 @@
 
 static int profile_file = -1;
 static long prepare_interval_usec;
+static struct profbuf_s *volatile current_codes;
 
-static int opened_profile(const char *interp_name);
+static int opened_profile(char *interp_name);
+static void flush_codes(void);
 
 RPY_EXTERN
-char *vmprof_init(int fd, double interval, const char *interp_name)
+char *vmprof_init(int fd, double interval, char *interp_name)
 {
     if (interval < 1e-6 || interval >= 1.0)
         return "bad value for 'interval'";
@@ -404,15 +406,15 @@
     return 0;
 }
 
-static int opened_profile(const char *interp_name)
+static int opened_profile(char *interp_name)
 {
     struct {
         long hdr[5];
         char interp_name[259];
     } header;
 
-    size_t namelen = strlen(interp_name);
-    assert(namelen <= 255);
+    size_t namelen = strnlen(interp_name, 255);
+    current_codes = NULL;
 
     header.hdr[0] = 0;
     header.hdr[1] = 3;
@@ -480,26 +482,76 @@
         return -1;
     if (remove_sigprof_handler() == -1)
         return -1;
+    flush_codes();
     if (shutdown_concurrent_bufs(profile_file) < 0)
         return -1;
     return close_profile();
 }
 
 RPY_EXTERN
-void vmprof_write_buf(char *buf, long size)
+int vmprof_register_virtual_function(char *code_name, long code_uid,
+                                     int auto_retry)
 {
+    long namelen = strnlen(code_name, 1023);
+    long blocklen = 1 + 2 * sizeof(long) + namelen;
     struct profbuf_s *p;
+    char *t;
 
-    while ((p = reserve_buffer(profile_file)) == NULL) {
-        /* spin loop waiting for a buffer to be ready; should almost never
-           be the case */
-        usleep(1);
+ retry:
+    p = current_codes;
+    if (p != NULL) {
+        if (__sync_bool_compare_and_swap(¤t_codes, p, NULL)) {
+            /* grabbed 'current_codes': we will append the current block
+               to it if it contains enough room */
+            size_t freesize = SINGLE_BUF_SIZE - p->data_size;
+            if (freesize < blocklen) {
+                /* full: flush it */
+                commit_buffer(profile_file, p);
+                p = NULL;
+            }
+        }
+        else {
+            /* compare-and-swap failed, don't try again */
+            p = NULL;
+        }
     }
 
-    if (size > SINGLE_BUF_SIZE)
-        size = SINGLE_BUF_SIZE;
-    memcpy(p->data, buf, size);
-    p->data_size = size;
+    if (p == NULL) {
+        p = reserve_buffer(profile_file);
+        if (p == NULL) {
+            /* can't get a free block; should almost never be the
+               case.  Spin loop if allowed, or return a failure code
+               if not (e.g. we're in a signal handler) */
+            if (auto_retry > 0) {
+                auto_retry--;
+                usleep(1);
+                goto retry;
+            }
+            return -1;
+        }
+    }
 
-    commit_buffer(profile_file, p);
+    t = p->data + p->data_size;
+    p->data_size += blocklen;
+    assert(p->data_size <= SINGLE_BUF_SIZE);
+    *t++ = MARKER_VIRTUAL_IP;
+    memcpy(t, &code_uid, sizeof(long)); t += sizeof(long);
+    memcpy(t, &namelen, sizeof(long)); t += sizeof(long);
+    memcpy(t, code_name, namelen);
+
+    /* try to reattach 'p' to 'current_codes' */
+    if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) {
+        /* failed, flush it */
+        commit_buffer(profile_file, p);
+    }
+    return 0;
 }
+
+static void flush_codes(void)
+{
+    struct profbuf_s *p = current_codes;
+    if (p != NULL) {
+        current_codes = NULL;
+        commit_buffer(profile_file, p);
+    }
+}
diff --git a/rpython/rlib/rvmprof/src/vmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
--- a/rpython/rlib/rvmprof/src/vmprof_mt.h
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -190,6 +190,13 @@
     }
 }
 
+static void cancel_buffer(struct profbuf_s *buf)
+{
+    long i = buf - profbuf_all_buffers;
+    assert(profbuf_state[i] == PROFBUF_FILLING);
+    profbuf_state[i] = PROFBUF_UNUSED;
+}
+
 static int shutdown_concurrent_bufs(int fd)
 {
     /* no signal handler can be running concurrently here, because we

From noreply at buildbot.pypy.org  Tue Aug 18 19:38:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 18 Aug 2015 19:38:16 +0200 (CEST)
Subject: [pypy-commit] pypy default: Reimport tweaks done for the CPython
	version
Message-ID: <20150818173816.81DBA1C0695@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79046:71f9a7fb1933
Date: 2015-08-18 19:32 +0200
http://bitbucket.org/pypy/pypy/changeset/71f9a7fb1933/

Log:	Reimport tweaks done for the CPython version

diff --git a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -7,12 +7,69 @@
 #endif
 
 
+#ifdef CPYTHON_GET_CUSTOM_OFFSET
+static void *tramp_start, *tramp_end;
+#endif
+
+
 static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
+
+#if defined(PYPY_JIT_CODEMAP)
+
     intptr_t ip_l = (intptr_t)ip;
     return pypy_jit_stack_depth_at_loc(ip_l);
+
+#elif defined(CPYTHON_GET_CUSTOM_OFFSET)
+
+    if (ip >= tramp_start && ip <= tramp_end) {
+        // XXX the return value is wrong for all the places before push and
+        //     after pop, fix
+        void *bp;
+        void *sp;
+
+        /* This is a stage2 trampoline created by hotpatch:
+
+               push   %rbx
+               push   %rbp
+               mov    %rsp,%rbp
+               and    $0xfffffffffffffff0,%rsp   // make sure the stack is aligned
+               movabs $0x7ffff687bb10,%rbx
+               callq  *%rbx
+               leaveq 
+               pop    %rbx
+               retq   
+
+           the stack layout is like this:
+
+               +-----------+                      high addresses
+               | ret addr  |
+               +-----------+
+               | saved rbx |   start of the function frame
+               +-----------+
+               | saved rbp |
+               +-----------+
+               | ........  |   <-- rbp
+               +-----------+                      low addresses
+
+           So, the trampoline frame starts at rbp+16, and the return address,
+           is at rbp+24.  The vmprof API requires us to return the offset of
+           the frame relative to sp, hence we have this weird computation.
+
+           XXX (antocuni): I think we could change the API to return directly
+           the frame address instead of the offset; however, this require a
+           change in the PyPy code too
+        */
+
+        unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
+        unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp);
+        return bp+16+8-sp;
+    }
+    return -1;
+
 #else
+
     return -1;
+
 #endif
 }
 
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -46,6 +46,7 @@
 static int profile_file = -1;
 static long prepare_interval_usec;
 static struct profbuf_s *volatile current_codes;
+static void *(*mainloop_get_virtual_ip)(char *) = 0;
 
 static int opened_profile(char *interp_name);
 static void flush_codes(void);
@@ -217,12 +218,10 @@
             // found main loop stack frame
             void* sp;
             unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-            void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
-            void **arg_ptr = (void**)arg_addr;
-            /* if (mainloop_get_virtual_ip) {
-               ip = mainloop_get_virtual_ip(*arg_ptr);
-               } else { */
-            ip = *arg_ptr;
+            if (mainloop_get_virtual_ip)
+                ip = mainloop_get_virtual_ip((char *)sp);
+            else
+                ip = *(void **)sp;
         }
 
         int first_run = (n == 0);
@@ -278,8 +277,8 @@
             st->stack[0] = GetPC((ucontext_t*)ucontext);
             depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-2, ucontext);
             depth++;  // To account for pc value in stack[0];
+            st->depth = depth;
             st->stack[depth++] = get_current_thread_id();
-            st->depth = depth;
             p->data_offset = offsetof(struct prof_stacktrace_s, marker);
             p->data_size = (depth * sizeof(void *) +
                             sizeof(struct prof_stacktrace_s) -

From noreply at buildbot.pypy.org  Tue Aug 18 22:34:33 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Aug 2015 22:34:33 +0200 (CEST)
Subject: [pypy-commit] buildbot coalesce-by-hash: test,
 fix coalescing builds with same hash, different hg revision numbers
Message-ID: <20150818203433.7C8D01C150B@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: coalesce-by-hash
Changeset: r958:f369232d9640
Date: 2015-08-18 23:35 +0300
http://bitbucket.org/pypy/buildbot/changeset/f369232d9640/

Log:	test, fix coalescing builds with same hash, different hg revision
	numbers

	also s/map/builds/ since map should not be a local variable

diff --git a/bot2/pypybuildbot/summary.py b/bot2/pypybuildbot/summary.py
--- a/bot2/pypybuildbot/summary.py
+++ b/bot2/pypybuildbot/summary.py
@@ -233,19 +233,23 @@
 
 class GatherOutcomeSet(object):
 
-    def __init__(self, map):
-        self.map = map
+    def __init__(self, builds):
+        self.builds = builds
         self._failed = None
         self._skipped = None
         self._numpassed = None
         self._numxfailed = None
-        self.revision = map.values()[0].revision
+        # In the case of a hg revision, this is of the form int:hash
+        # where the hashes should be identical, the int values may
+        # vary dependent on build slave repe. 
+        # Should we sort, if so by what?
+        self.revision = builds.values()[0].revision
 
     @property
     def failed(self):
         if self._failed is None:
             self._failed = set()
-            for prefix, outcome in self.map.items():
+            for prefix, outcome in self.builds.items():
                 self._failed.update([(prefix,)+ namekey for namekey in
                                      outcome.failed])
         return self._failed
@@ -254,7 +258,7 @@
     def skipped(self):
         if self._skipped is None:
             self._skipped = set()
-            for prefix, outcome in self.map.items():
+            for prefix, outcome in self.builds.items():
                 self._skipped.update([(prefix,) + namekey for namekey in
                                      outcome.skipped])
         return self._skipped
@@ -263,7 +267,7 @@
     def numpassed(self):
         if self._numpassed is None:
             numpassed = 0
-            for  prefix, outcome in self.map.items():
+            for  prefix, outcome in self.builds.items():
                 numpassed += outcome.numpassed
             self._numpassed = numpassed
         return self._numpassed
@@ -272,29 +276,29 @@
     def numxfailed(self):
         if self._numxfailed is None:
             numxfailed = 0
-            for  prefix, outcome in self.map.items():
+            for  prefix, outcome in self.builds.items():
                 numxfailed += outcome.numxfailed
             self._numxfailed = numxfailed
         return self._numxfailed
 
     def get_outcome(self, namekey):
         which = namekey[0]
-        if which not in self.map:
+        if which not in self.builds:
             return ' '
-        return self.map[which].get_outcome(namekey[1:])
+        return self.builds[which].get_outcome(namekey[1:])
 
     def get_longrepr(self, namekey):
         which = namekey[0]
-        if which not in self.map:
+        if which not in self.builds:
             return ''
-        return self.map[which].get_longrepr(namekey[1:])
+        return self.builds[which].get_longrepr(namekey[1:])
 
     def get_key_namekey(self, namekey):
-        return self.map[namekey[0]].get_key_namekey(namekey[1:])
+        return self.builds[namekey[0]].get_key_namekey(namekey[1:])
 
     def get_run_infos(self):
         all = {}
-        for outcome_set in self.map.itervalues():
+        for outcome_set in self.builds.itervalues():
             all.update(outcome_set.get_run_infos())
         return all
 
@@ -403,7 +407,7 @@
                       class_=' '.join(["failSummary", cls]))
 
     def _builder_num(self, outcome_set):
-        return outcome_set.map.values()[0].key
+        return outcome_set.builds.values()[0].key
 
     def _label(self, outcome_set):
         if self.fixed_builder:
@@ -429,7 +433,7 @@
         if self.fixed_builder:
             pick = "builder=%s&builds=%d" % self._builder_num(outcome_set)
         else:
-            pick = "recentrev=%s" % rev
+            pick = "recentrev=%s" % rev.split(':')[-1]
         category, branch = self.cur_cat_branch
         revtxt = str(rev)
         rev_anchor = html.a(revtxt, href="/summary?category=%s&branch=%s&%s" %
@@ -752,6 +756,8 @@
                 if not test_branch(branch):
                     continue
                 got_rev = getProp(build, 'got_revision', None)
+                if got_rev:
+                    got_rev = got_rev.split(':')[-1]
                 if not test_rev(got_rev):
                     continue
 
@@ -766,6 +772,8 @@
                         no_revision_builds.append(build)
                 else:
                     rev = got_rev
+                    if ':' in rev:
+                        rev = rev.split(':')[-1]
                     buildNumber = build.getNumber()
                     if fixed_builder:
                         builds = runs.setdefault((buildNumber, rev), {})
@@ -783,7 +791,6 @@
                     key = (builderName, buildNumber)
                     outcome_set = outcome_set_cache.get(status, key)
                     runBuilds[builderName] = outcome_set
-
         return cat_branches
 
     @staticmethod
diff --git a/bot2/pypybuildbot/test/test_summary.py b/bot2/pypybuildbot/test/test_summary.py
--- a/bot2/pypybuildbot/test/test_summary.py
+++ b/bot2/pypybuildbot/test/test_summary.py
@@ -797,23 +797,31 @@
         builder1 = status_builder.BuilderStatus('builder_foo', 'foo', self.master, '')
         builder2 = status_builder.BuilderStatus('builder_bar', 'bar', self.master, '')
         builder3 = status_builder.BuilderStatus('builder_', '', self.master, '')
+        builder4 = status_builder.BuilderStatus('builder_xyz', 'foo', self.master, '')
 
         add_builds(builder1, [('60000', "F TEST1\n")])
         add_builds(builder2, [('60000', "F TEST2\n")])
         add_builds(builder3, [('60000', "F TEST3\n")])
+        add_builds(builder4, [('60000', "F TEST4\n")])
 
         s = summary.Summary(['foo', 'bar'])
-        req = FakeRequest([builder1, builder2, builder3])
+        req = FakeRequest([builder1, builder2, builder3, builder4])
         out = s.body(req)
 
         rel1 = out.index('TEST1')
         rel2 = out.index('TEST2')
         rel3 = out.index('TEST3')
+        rel4 = out.index('TEST4')
 
-        assert rel3 > rel2 > rel1
+        assert rel3 > rel2 > rel4 > rel1
 
-        assert "{foo}" in out
-        assert "{bar}" in out
+        # since the two foo category builds have the same rev number, they
+        # should appear on the same line. Strip HTML tags and check for the rev
+        plain = re.sub('<[^<]+?>', '', out)
+        relFoo = plain.index("{foo}")
+        relBar = plain.index("{bar}")
+        
+        assert plain.count('60000', relFoo, relBar) == 1
 
     def test_two_builds_different_rev_digits(self):
         builder = status_builder.BuilderStatus('builder0', '', self.master, '')
@@ -829,6 +837,32 @@
         p1000builder0 = out.find('builder0', p1000)
         assert p999builder0-p999 == p1000builder0-p1000+1
 
+    def test_builds_coalesced_by_rev_hash(self):
+        # Two builds with the same category and the same hash should ignore
+        # revision numbers
+        category = 'nightly'
+        builder0 = status_builder.BuilderStatus('builder1', category, self.master, '')
+        builder1 = status_builder.BuilderStatus('builder0', category, self.master, '')
+        add_builds(builder0, [('999:abcdefg', "F TEST1\n. b"),
+                             ('1000:hijklmn', "F TEST1\n. b")])
+        add_builds(builder1, [('1999:abcdefg', "F TEST3\n. b"),
+                             ('2000:hijklmn', "F TEST3\n. b")])
+
+        s = summary.Summary()
+        req = FakeRequest([builder0, builder1])
+        out = s.body(req)
+
+        p999 = out.find('999')
+        p999builder0 = out.find('builder0', p999)
+        p1000 = out.find('1000')
+        p1000builder0 = out.find('builder0', p1000)
+        assert p999builder0-p999 == p1000builder0-p1000+2,'%d != %d' %(
+                                        p999builder0-p999, p1000builder0-p1000+1)
+
+        req.args = {'recentrev': ['abcdefg']}
+        out = s.body(req)
+        assert out.count('999') == 1
+
     def test_build_times_and_filtering(self):
         builder1 = status_builder.BuilderStatus('builder1', '', self.master, '')
         builder2 = status_builder.BuilderStatus('builder2', '', self.master, '')

From noreply at buildbot.pypy.org  Tue Aug 18 23:33:56 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Aug 2015 23:33:56 +0200 (CEST)
Subject: [pypy-commit] buildbot default: use a string, not a list,
 so shell expansion will work
Message-ID: <20150818213356.93D141C1E82@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r959:80cd482cd79e
Date: 2015-08-19 00:33 +0300
http://bitbucket.org/pypy/buildbot/changeset/80cd482cd79e/

Log:	use a string, not a list, so shell expansion will work

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -672,7 +672,7 @@
             workdir='.'))
         self.addStep(ShellCmd(
             description="copy cffi import libraries",
-            command=['cp', '-rv', 'pypy-c/lib_pypy/*.so', 'build/lib_pypy'],
+            command='cp -rv pypy-c/lib_pypy/*.so build/lib_pypy',
             haltOnFailure=True,
             workdir='.'))
 

From noreply at buildbot.pypy.org  Tue Aug 18 23:33:58 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 18 Aug 2015 23:33:58 +0200 (CEST)
Subject: [pypy-commit] buildbot coalesce-by-hash: merge deault into branch
Message-ID: <20150818213358.8844C1C1EF2@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: coalesce-by-hash
Changeset: r960:39eafa031c8e
Date: 2015-08-19 00:34 +0300
http://bitbucket.org/pypy/buildbot/changeset/39eafa031c8e/

Log:	merge deault into branch

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -672,7 +672,7 @@
             workdir='.'))
         self.addStep(ShellCmd(
             description="copy cffi import libraries",
-            command=['cp', '-rv', 'pypy-c/lib_pypy/*.so', 'build/lib_pypy'],
+            command='cp -rv pypy-c/lib_pypy/*.so build/lib_pypy',
             haltOnFailure=True,
             workdir='.'))
 

From noreply at buildbot.pypy.org  Wed Aug 19 11:09:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 11:09:20 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: some docs about unrolling
Message-ID: <20150819090920.845201C0276@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79047:ccf6168de8e8
Date: 2015-08-19 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ccf6168de8e8/

Log:	some docs about unrolling

diff --git a/rpython/jit/metainterp/optimizeopt/unrolling.rst b/rpython/jit/metainterp/optimizeopt/unrolling.rst
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/unrolling.rst
@@ -0,0 +1,42 @@
+
+nomenclature:
+
+Unrolling follows that order of operations:
+
+* optimize one version of a loop, call it . At the end of ,
+  we gather virtual state by visiting the status of optimized boxes as
+  presented in the arguments to JUMP. Virtual state then can produce
+  a number of inputargs from a list
+
+* gather all the operations that can be produced in the short preamble. They're,
+  as far as I can tell:
+
+  - jump arguments, as generated from the virtual state
+
+  - pure operations, including pure calls
+
+  - call_loopinvariant
+
+  - heap cache
+
+  or the combination of the above. Note that some operations can be produced
+  in more than one way. In those cases we keep both.
+
+* the optimizer state is influenced by replaying all the short boxes producing
+  operations. This populates the heap cache and the pure operation cache.
+
+* import the values of the inputargs (so e.g. constantness, intbounds, etc.)
+
+* we optimize normally, using extra boxes from the optimizer state
+
+* we import all the boxes that were not generated during this optimization
+  phase. This means, they have to:
+
+  * be added to the start label
+
+  * be added to the end jump
+
+  * be added to the short preamble
+
+* we close the loop, which creates correct boxes in the jump (from virtualstate)
+  

From noreply at buildbot.pypy.org  Wed Aug 19 11:09:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 11:09:22 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start working on direct tests
	for bridges
Message-ID: <20150819090922.A34401C0276@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79048:2f2f6319cdd2
Date: 2015-08-19 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/2f2f6319cdd2/

Log:	start working on direct tests for bridges

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -76,6 +76,26 @@
         return opt.propagate_all_forward(self.start_label.getarglist(),
             self.operations, self.call_pure_results, self.enable_opts)
 
+class BridgeCompileData(CompileData):
+    """ This represents ops() with a jump at the end that goes to some
+    loop, we need to deal with virtual state and inlining of short preamble
+    """
+    def __init__(self, start_label, operations, call_pure_results=None,
+                 enable_opts=None, inline_short_preamble=False):
+        self.start_label = start_label
+        self.operations = operations
+        self.call_pure_results = call_pure_results
+        self.enable_opts = enable_opts
+        self.inline_short_preamble = inline_short_preamble
+
+    def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
+        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+
+        opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
+        return opt.optimize_bridge(self.start_label, self.operations,
+                                   self.call_pure_results,
+                                   self.inline_short_preamble)
+
 class UnrolledLoopData(CompileData):
     """ This represents label() ops jump with extra info that's from the
     run of LoopCompileData. Jump goes to the same label
@@ -253,7 +273,7 @@
     loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
                        [mid_label] + loop_ops)
     loop.check_consistency()
-    jitcell_token.all_target_tokens = [start_descr, mid_descr_token]
+    jitcell_token.target_tokens = [mid_descr_token, start_descr]
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
     return start_descr
@@ -783,10 +803,10 @@
         # to the corresponding guard_op and compile from there
         assert metainterp.resumekey_original_loop_token is not None
         new_loop.original_jitcell_token = metainterp.resumekey_original_loop_token
-        inputargs = metainterp.history.inputargs
+        inputargs = new_loop.inputargs
         if not we_are_translated():
             self._debug_suboperations = new_loop.operations
-        propagate_original_jitcell_token(new_loop)
+        #propagate_original_jitcell_token(new_loop)
         send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata,
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
@@ -1001,11 +1021,10 @@
     metainterp_sd = metainterp.staticdata
     jitdriver_sd = metainterp.jitdriver_sd
     state = jitdriver_sd.warmstate
-    #if isinstance(resumekey, ResumeAtPositionDescr):
-    #    inline_short_preamble = False
-    #else:
-    #    xxx
-    #    inline_short_preamble = True
+    if isinstance(resumekey, ResumeAtPositionDescr):
+        inline_short_preamble = False
+    else:
+        inline_short_preamble = True
     inputargs = metainterp.history.inputargs[:]
     operations = metainterp.history.operations
     label = ResOperation(rop.LABEL, inputargs)
@@ -1014,9 +1033,15 @@
 
     call_pure_results = metainterp.call_pure_results
 
-    data = SimpleCompileData(label, operations,
-                             call_pure_results=call_pure_results,
-                             enable_opts=enable_opts)
+    if operations[-1].getopnum() == rop.JUMP:
+        data = BridgeCompileData(label, operations[:],
+                                 call_pure_results=call_pure_results,
+                                 enable_opts=enable_opts,
+                                 inline_short_preamble=inline_short_preamble)
+    else:
+        data = SimpleCompileData(label, operations[:],
+                                 call_pure_results=call_pure_results,
+                                 enable_opts=enable_opts)
     try:
         info, newops = optimize_trace(metainterp_sd, jitdriver_sd, data)
     except InvalidLoop:
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -521,6 +521,10 @@
     def _emit_operation(self, op):
         assert not op.is_call_pure()
         orig_op = op
+        op = self.get_box_replacement(op)
+        if op.is_constant():
+            return # can happen e.g. if we postpone the operation that becomes
+            # constant
         op = self.replace_op_with(op, op.getopnum())
         for i in range(op.numargs()):
             arg = self.force_box(op.getarg(i))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -0,0 +1,56 @@
+
+from rpython.jit.metainterp.optimizeopt.test.test_util import BaseTest,\
+     LLtypeMixin, convert_old_style_to_targets
+from rpython.jit.metainterp import compile
+from rpython.jit.metainterp.resoperation import ResOperation, rop
+from rpython.jit.metainterp.history import TargetToken
+
+class TestOptimizeBridge(BaseTest, LLtypeMixin):
+    enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
+
+    def optimize(self, ops, bridge_ops, expected, inline_short_preamble=True):
+        loop = self.parse(ops, postprocess=self.postprocess)
+        info = self.unroll_and_optimize(loop, None)
+        jitcell_token = compile.make_jitcell_token(None)
+        mid_label_descr = TargetToken(jitcell_token)
+        start_label_descr = TargetToken(jitcell_token)
+        jitcell_token.target_tokens = [mid_label_descr, start_label_descr]
+        loop.operations[0].setdescr(mid_label_descr)
+        loop.operations[-1].setdescr(mid_label_descr)
+        info.preamble.operations[0].setdescr(start_label_descr)
+        guards = [op for op in loop.operations if op.is_guard()]
+        assert len(guards) == 1, "more than one guard in the loop"
+        bridge = self.parse(bridge_ops, postprocess=self.postprocess)
+        start_label = ResOperation(rop.LABEL, bridge.inputargs)
+        bridge.operations[-1].setdescr(jitcell_token)
+        data = compile.BridgeCompileData(start_label, bridge.operations,
+                                         enable_opts=self.enable_opts,
+                            inline_short_preamble=inline_short_preamble)
+        bridge_info, ops = self._do_optimize_loop(data)
+        loop.check_consistency(check_descr=False)
+        info.preamble.check_consistency(check_descr=False)
+        bridge.operations = ([ResOperation(rop.LABEL, bridge_info.inputargs)] +
+                             ops)
+        bridge.inputargs = bridge_info.inputargs
+        bridge.check_consistency(check_descr=False)
+        expected = self.parse(expected, postprocess=self.postprocess)
+        self.assert_equal(bridge, convert_old_style_to_targets(expected,
+                                                               jump=True))
+    
+    def test_simple(self):
+        loop = """
+        [i0]
+        i1 = int_add(i0, 1)
+        i2 = int_is_true(i1)
+        guard_true(i2) [i1, i2]
+        jump(i1)
+        """
+        bridge = """
+        [i0, i1]
+        jump(i1)
+        """
+        expected = """
+        [i0, i1]
+        jump(i1)
+        """
+        self.optimize(loop, bridge, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -122,6 +122,23 @@
                            self.short_preamble_producer.extra_same_as),
                 self.optimizer._newoperations)
 
+    def optimize_bridge(self, start_label, operations, call_pure_results,
+                        inline_short_preamble):
+        assert inline_short_preamble
+        info, ops = self.optimizer.propagate_all_forward(
+            start_label.getarglist()[:], operations[:-1],
+            call_pure_results, True)
+        jump_op = operations[-1]
+        self.jump_to_existing_trace(jump_op, inline_short_preamble)
+        return info, self.optimizer._newoperations[:]
+
+    def jump_to_existing_trace(self, jump_op, inline_short_preamble):
+        jitcelltoken = jump_op.getdescr()
+        args = jump_op.getarglist()
+        virtual_state = self.get_virtual_state(args)
+        self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
+                                  descr=jitcelltoken.target_tokens[0]))
+
     def filter_extra_jump_args(self, label_args, jump_args):
         new_label_args = []
         new_jump_args = []

From noreply at buildbot.pypy.org  Wed Aug 19 11:17:30 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 11:17:30 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: write a minimal test for short
 preamble inlining
Message-ID: <20150819091730.B3EBE1C0276@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79049:84d61e80af51
Date: 2015-08-19 11:17 +0200
http://bitbucket.org/pypy/pypy/changeset/84d61e80af51/

Log:	write a minimal test for short preamble inlining

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -8,8 +8,13 @@
 class TestOptimizeBridge(BaseTest, LLtypeMixin):
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
-    def optimize(self, ops, bridge_ops, expected, inline_short_preamble=True):
+    def optimize(self, ops, bridge_ops, expected, expected_loop=None,
+                 inline_short_preamble=True):
         loop = self.parse(ops, postprocess=self.postprocess)
+        if expected_loop is not None:
+            xxx
+            exp_loop = self.parse(expected_loop, postprocess=self.postprocess)
+            self.assert_equal(loop, convert_old_style_to_targets(exp_loop))
         info = self.unroll_and_optimize(loop, None)
         jitcell_token = compile.make_jitcell_token(None)
         mid_label_descr = TargetToken(jitcell_token)
@@ -36,6 +41,18 @@
         expected = self.parse(expected, postprocess=self.postprocess)
         self.assert_equal(bridge, convert_old_style_to_targets(expected,
                                                                jump=True))
+        jump_bridge = bridge.operations[-1]
+        jump_d = jump_bridge.getdescr()
+        jump_args = jump_bridge.getarglist()
+        if loop.operations[0].getdescr() is jump_d:
+            # jump to loop
+            label_args = loop.operations[0].getarglist()
+        else:
+            assert info.preamble.operations[0].getdescr() is jump_d
+            label_args = info.preamble.operations[0].getarglist()
+        assert len(jump_args) == len(label_args)
+        for a, b in zip(jump_args, label_args):
+            assert a.type == b.type
     
     def test_simple(self):
         loop = """
@@ -54,3 +71,18 @@
         jump(i1)
         """
         self.optimize(loop, bridge, expected)
+
+    def test_minimal_short_preamble(self):
+        loop = """
+        [i0, i1, i3]
+        i2 = int_add(i0, 1)
+        i4 = int_add(i3, i2)
+        i5 = int_is_true(i4)
+        guard_true(i5) [i2, i4, i5]
+        jump(i0, i1, i4)
+        """
+        bridge = """
+        [i0, i1, i2]
+        jump(i0, i1, i2)
+        """
+        self.optimize(loop, bridge, bridge)

From noreply at buildbot.pypy.org  Wed Aug 19 11:55:01 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 19 Aug 2015 11:55:01 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: testing the path iteration,
 there is still a problem in analyse_index_calculation
Message-ID: <20150819095501.75A531C1330@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79050:a376c01d579f
Date: 2015-08-19 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/a376c01d579f/

Log:	testing the path iteration, there is still a problem in
	analyse_index_calculation

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -112,6 +112,7 @@
         # save the operation that produces the result for the first argument
         # only for guard_true/guard_false
         self.guard_bool_bool_node = None
+        self._stack = False
 
     def getoperation(self):
         return self.op
@@ -285,28 +286,31 @@
             else:
                 iterdir = node.provides()
             if index >= len(iterdir):
-                if blacklist:
-                    blacklist_visit[node] = None
+                #if blacklist:
+                    #blacklist_visit[node] = None
+                #    print "blacklisting 1", node, "path", '->'.join([str(p.opidx) for p in path.path])
                 continue
             else:
                 next_dep = iterdir[index]
                 next_node = next_dep.to
                 index += 1
+                if index < len(iterdir):
+                    worklist.append((index, node, pathlen))
+                else:
+                    print "blacklisting 2", node, "path", '->'.join([str(p.opidx) for p in path.path])
+                    blacklist_visit[node] = None
+                path.cut_off_at(pathlen)
+                path.walk(next_node)
                 if blacklist and next_node in blacklist_visit:
                     yield Path(path.path[:])
                     continue
-                if index < len(iterdir):
-                    worklist.append((index, node, pathlen))
-                else:
-                    blacklist_visit[node] = None
-                path.cut_off_at(pathlen)
-                path.walk(next_node)
                 pathlen += 1
 
                 if next_node is to or (path_max_len > 0 and pathlen >= path_max_len):
                     yield Path(path.path[:])
-                    if blacklist:
-                        blacklist_visit[next_node] = None
+                    # note that the destiantion node ``to'' is never blacklisted
+                    #if blacklist:
+                    #    blacklist_visit[next_node] = None
                 else:
                     worklist.append((0, next_node, pathlen))
 
@@ -731,6 +735,17 @@
             # and the next guard instruction
             tracker.add_non_pure(node)
 
+    def cycles(self):
+        """ NOT_RPYTHON """
+        stack = []
+        for node in self.nodes:
+            node._stack = False
+        #
+        label = self.nodes[0]
+        if _first_cycle(stack, label):
+            return stack
+        return None
+
     def __repr__(self):
         graph = "graph([\n"
         for node in self.nodes:
@@ -744,6 +759,7 @@
         return graph + "      ])"
 
     def as_dot(self):
+        """ NOT_RPTYHON """
         if not we_are_translated():
             dot = "digraph dep_graph {\n"
             for node in self.nodes:
@@ -765,6 +781,49 @@
             return dot
         raise NotImplementedError("dot only for debug purpose")
 
+def _first_cycle(stack, node):
+    node._stack = True
+    stack.append(node)
+
+    for dep in node.provides():
+        succ = dep.to
+        if succ._stack:
+            # found cycle!
+            while stack[0] is not succ:
+                del stack[0]
+            return True
+        else:
+            return _first_cycle(stack, succ)
+
+    return False
+
+def _strongly_connect(index, stack, cycles, node):
+    """ currently unused """
+    node._scc_index = index
+    node._scc_lowlink = index
+    index += 1
+    stack.append(node)
+    node._scc_stack = True
+
+    for dep in node.provides():
+        succ = dep.to
+        if succ._scc_index == -1:
+            index = _strongly_connect(index, stack, cycles, succ)
+            node._scc_lowlink = min(node._scc_lowlink, succ._scc_lowlink)
+        elif succ._scc_stack:
+            node._scc_lowlink = min(node._scc_lowlink, succ._scc_index)
+
+    if node._scc_lowlink == node._scc_index:
+        cycle = []
+        while True:
+            w = stack.pop()
+            w._scc_stack = False
+            cycle.append(w)
+            if w is node:
+                break
+        cycles.append(cycle)
+    return index
+
 class IntegralForwardModification(object):
     """ Calculates integral modifications on integer boxes. """
     def __init__(self, memory_refs, index_vars, comparison_vars, invariant_vars):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -17,6 +17,10 @@
     def __repr__(self):
         return "n%d" % self.opidx
 
+class FakeDependencyGraph(DependencyGraph):
+    def __init__(self):
+        pass
+
 class DependencyBaseTest(BaseTest):
 
     def setup_method(self, method):
@@ -391,7 +395,7 @@
         self.assert_dependencies(trace, full_check=True)
 
 
-    def test_iterate_paths(self):
+    def test_iterate(self):
         n1,n2,n3,n4,n5 = [FakeNode(i+1) for i in range(5)]
         # n1 -> n2 -> n4 -> n5
         #  +---> n3 --^
@@ -434,7 +438,7 @@
         for i in r:
             assert paths[i].as_str() == "n0 -> %s -> n%d" % (nodes[i], len(r)+1)
 
-    def test_iterate_paths_blacklist_diamond(self):
+    def test_iterate_blacklist_diamond(self):
         blacklist = {}
         n1,n2,n3,n4 = [FakeNode(i+1) for i in range(4)]
         # n1 -> n2 -> n4
@@ -445,9 +449,9 @@
         paths = list(n1.iterate_paths(n4, blacklist=True))
         assert len(paths) == 2
         assert paths[0].as_str() == "n1 -> n2 -> n4"
-        assert paths[1].as_str() == "n1 -> n3"
+        assert paths[1].as_str() == "n1 -> n3 -> n4"
 
-    def test_iterate_paths_blacklist_double_diamond(self):
+    def test_iterate_blacklist_double_diamond(self):
         blacklist = {}
         n1,n2,n3,n4,n5,n6,n7,n8 = [FakeNode(i+1) for i in range(8)]
         # n1 -> n2 -> n4 -> n5 -> n6 --> n8
@@ -461,8 +465,56 @@
         paths = list(n1.iterate_paths(n8, blacklist=True))
         assert len(paths) == 3
         assert paths[0].as_str() == "n1 -> n2 -> n4 -> n5 -> n6 -> n8"
-        assert paths[1].as_str() == "n1 -> n2 -> n4 -> n5 -> n7"
-        assert paths[2].as_str() == "n1 -> n3"
+        assert paths[1].as_str() == "n1 -> n2 -> n4 -> n5 -> n7 -> n8"
+        assert paths[2].as_str() == "n1 -> n3 -> n4"
+
+    def test_iterate_blacklist_split_path(self):
+        blacklist = {}
+        n1,n2,n3,n4,n5,n6,n7,n8 = [FakeNode(i+1) for i in range(8)]
+        n1.edge_to(n2);
+        n3.edge_to(n2);
+        n2.edge_to(n4);
+        n3.edge_to(n4);
+
+        paths = list(n4.iterate_paths(n3, backwards=True, blacklist=True))
+        assert len(paths) == 2
+        assert paths[0].as_str() == "n4 -> n2 -> n3"
+        assert paths[1].as_str() == "n4 -> n3"
+
+        n5.edge_to(n1)
+        n5.edge_to(n3)
+
+        paths = list(n4.iterate_paths(n5, backwards=True, blacklist=True))
+        assert len(paths) == 3
+        assert paths[0].as_str() == "n4 -> n2 -> n1 -> n5"
+        assert paths[1].as_str() == "n4 -> n2 -> n3 -> n5"
+        assert paths[2].as_str() == "n4 -> n3"
+
+    def test_sccs(self):
+        n1,n2 = FakeNode(1), FakeNode(2)
+        n1.edge_to(n2); n2.edge_to(n1)
+
+        graph = FakeDependencyGraph()
+        graph.nodes = [n1,n2]
+        cycle = graph.cycles()
+        assert cycle == [n1, n2]
+
+        n3 = FakeNode(0)
+        graph.nodes = [n3]
+        cycle = graph.cycles()
+        assert cycle is None
+
+    def test_cycles_2(self):
+        n1,n2,n3,n4 = FakeNode(1), FakeNode(2), FakeNode(3), FakeNode(4)
+        n1.edge_to(n3); n3.edge_to(n4); n4.edge_to(n1)
+
+        graph = FakeDependencyGraph()
+        graph.nodes = [n1,n2,n3]
+        cycle = graph.cycles()
+        assert cycle is not None
+        assert cycle == [n1,n3,n4]
+
+
 
 class TestLLtype(BaseTestDependencyGraph, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -93,6 +93,9 @@
         self.show_dot_graph(DependencyGraph(opt.loop), "original_" + self.test_name)
         opt.analyse_index_calculations()
         if opt.dependency_graph is not None:
+            cycle = opt.dependency_graph.cycles()
+            if cycle is not None:
+                print "CYCLE found %s" % cycle
             self.show_dot_graph(opt.dependency_graph, "early_exit_" + self.test_name)
             opt.schedule(False)
         opt.unroll_loop_iterations(loop, unroll_factor)
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -515,23 +515,27 @@
         label_node = graph.getnode(0)
         ee_guard_node = graph.getnode(ee_pos)
         guards = graph.guards
+        unique = set()
         for guard_node in guards:
             if guard_node is ee_guard_node:
                 continue
             modify_later = []
             last_prev_node = None
-            i = 0
             for path in guard_node.iterate_paths(ee_guard_node, backwards=True, blacklist=True):
+                p = '->'.join([str(p.opidx) for p in path.path])
+                if p in unique:
+                    assert 0
+                else:
+                    unique.add(p)
+                print "PATH:", p
                 if not we_are_translated():
                     path.check_acyclic()
-                print "loop", i
-                i+=1
                 prev_node = path.second()
                 dep = prev_node.depends_on(guard_node)
                 if dep.is_failarg():
                     # this dependency we are able to break because it is soley
                     # relevant due to one or multiple fail args
-                    if prev_node == last_prev_node:
+                    if prev_node is not last_prev_node:
                         #  ...
                         #  o  o
                         #  \ /
@@ -540,18 +544,19 @@
                         #  (g)
                         # this graph yields 2 paths from (g), thus (a) is
                         # remembered and skipped the second time visited
-                        continue
-                    modify_later.append((prev_node, guard_node))
+                        modify_later.append((prev_node, guard_node))
+                        print "  => remove guard -> second"
+                    last_prev_node = prev_node
+                    continue
+                if path.is_always_pure(exclude_first=True, exclude_last=True):
+                    path.set_schedule_priority(10)
+                    if path.last() is ee_guard_node:
+                        modify_later.append((path.last_but_one(), None))
+                        print "  => always pure"
                 else:
-                    if path.is_always_pure(exclude_first=True, exclude_last=True):
-                        path.set_schedule_priority(10)
-                        if path.last() is ee_guard_node:
-                            modify_later.append((path.last_but_one(), None))
-                    else:
-                        # transformation is invalid.
-                        # exit and do not enter else branch!
-                        break
-                last_prev_node = prev_node
+                    # transformation is invalid.
+                    # exit and do not enter else branch!
+                    break
             else:
                 # transformation is valid, modify the graph and execute
                 # this guard earlier
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -304,6 +304,8 @@
         self.vec = bool(value)
 
     def set_param_vec_params(self, value):
+        if NonConstant(False):
+            value = 'blah' # not a constant ''
         values = value.split(":")
         self.vec_all = bool(values[0])
         self.vec_cost = 0

From noreply at buildbot.pypy.org  Wed Aug 19 12:18:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 12:18:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: pass the first inlining of
 short preamble in the bridge, yay!
Message-ID: <20150819101841.07F5D1C1186@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79051:73dadee30d14
Date: 2015-08-19 12:18 +0200
http://bitbucket.org/pypy/pypy/changeset/73dadee30d14/

Log:	pass the first inlining of short preamble in the bridge, yay!

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -266,6 +266,8 @@
                                descr=start_descr)
     mid_descr_token = TargetToken(jitcell_token,
                                   original_jitcell_token=jitcell_token)
+    mid_descr_token.short_preamble = loop_info.short_preamble
+    mid_descr_token.virtual_state = start_state.virtual_state
     mid_label = ResOperation(rop.LABEL, loop_info.label_args,
                              descr=mid_descr_token)
     # XXX assign short preamble and virtual state
@@ -1043,10 +1045,12 @@
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
-        info, newops = optimize_trace(metainterp_sd, jitdriver_sd, data)
+        try:
+            info, newops = optimize_trace(metainterp_sd, jitdriver_sd, data)
+        finally:
+            forget_optimization_info(inputargs)
+            forget_optimization_info(operations)
     except InvalidLoop:
-        forget_optimization_info(inputargs)
-        forget_optimization_info(operations)
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
         # InvalidLoop
@@ -1056,9 +1060,9 @@
     new_trace = create_empty_loop(metainterp)
     new_trace.inputargs = info.inputargs
     new_trace.operations = newops
+    target_token = new_trace.operations[-1].getdescr()
     resumekey.compile_and_attach(metainterp, new_trace)
     record_loop_or_bridge(metainterp_sd, new_trace)
-    target_token = new_trace.operations[-1].getdescr()
     return target_token
     xxxx
     if new_trace.operations[-1].getopnum() != rop.LABEL:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -18,6 +18,8 @@
         info = self.unroll_and_optimize(loop, None)
         jitcell_token = compile.make_jitcell_token(None)
         mid_label_descr = TargetToken(jitcell_token)
+        mid_label_descr.short_preamble = info.short_preamble
+        mid_label_descr.virtual_state = info.virtual_state
         start_label_descr = TargetToken(jitcell_token)
         jitcell_token.target_tokens = [mid_label_descr, start_label_descr]
         loop.operations[0].setdescr(mid_label_descr)
@@ -28,6 +30,7 @@
         bridge = self.parse(bridge_ops, postprocess=self.postprocess)
         start_label = ResOperation(rop.LABEL, bridge.inputargs)
         bridge.operations[-1].setdescr(jitcell_token)
+        self.add_guard_future_condition(bridge)
         data = compile.BridgeCompileData(start_label, bridge.operations,
                                          enable_opts=self.enable_opts,
                             inline_short_preamble=inline_short_preamble)
@@ -85,4 +88,9 @@
         [i0, i1, i2]
         jump(i0, i1, i2)
         """
-        self.optimize(loop, bridge, bridge)
+        expected = """
+        [i0, i1, i2]
+        i3 = int_add(i0, 1)
+        jump(i0, i1, i2, i3)
+        """
+        self.optimize(loop, bridge, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -395,9 +395,10 @@
             return ""
 
 class Info(object):
-    def __init__(self, preamble, short_preamble=None):
+    def __init__(self, preamble, short_preamble=None, virtual_state=None):
         self.preamble = preamble
         self.short_preamble = short_preamble
+        self.virtual_state = virtual_state
 
 class Storage(compile.ResumeGuardDescr):
     "for tests."
@@ -494,7 +495,8 @@
                                loop_info.extra_same_as + [emit_end_label])
         loop.inputargs = inputargs
         loop.operations = [emit_end_label] + ops
-        return Info(preamble, loop_info.short_preamble)
+        return Info(preamble, loop_info.short_preamble,
+                    start_state.virtual_state)
 
 
 class FakeDescr(compile.ResumeGuardDescr):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -79,11 +79,12 @@
         modifier = VirtualStateConstructor(self.optimizer)
         return modifier.get_virtual_state(args)
 
-    def _check_no_forwarding(self, lsts):
+    def _check_no_forwarding(self, lsts, check_newops=True):
         for lst in lsts:
             for op in lst:
                 assert op.get_forwarded() is None
-        assert not self.optimizer._newoperations
+        if check_newops:
+            assert not self.optimizer._newoperations
     
     def optimize_preamble(self, start_label, end_label, ops, call_pure_results):
         self._check_no_forwarding([[start_label, end_label], ops])
@@ -110,7 +111,12 @@
         pass_to_short = state.virtual_state.make_inputargs(orig_jump_args,
                                     self.optimizer, force_boxes=True,
                                     append_virtuals=True)
-        extra_jump_args = self.inline_short_preamble(pass_to_short)
+        sb = self.short_preamble_producer
+        self.optimizer._clean_optimization_info(sb.short_inputargs)
+        extra_jump_args = self.inline_short_preamble(pass_to_short,
+                                sb.short_inputargs, sb.short,
+                                sb.short_preamble_jump,
+                                self.optimizer.patchguardop)
         # remove duplicates, removes stuff from used boxes too
         label_args, jump_args = self.filter_extra_jump_args(
             start_label.getarglist() + self.short_preamble_producer.used_boxes,
@@ -125,6 +131,8 @@
     def optimize_bridge(self, start_label, operations, call_pure_results,
                         inline_short_preamble):
         assert inline_short_preamble
+        self._check_no_forwarding([start_label.getarglist(),
+                                    operations])
         info, ops = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], operations[:-1],
             call_pure_results, True)
@@ -134,9 +142,15 @@
 
     def jump_to_existing_trace(self, jump_op, inline_short_preamble):
         jitcelltoken = jump_op.getdescr()
-        args = jump_op.getarglist()
+        args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
+        target_token = jitcelltoken.target_tokens[0]
         virtual_state = self.get_virtual_state(args)
+        short_preamble = target_token.short_preamble
+        extra = self.inline_short_preamble(args,
+            short_preamble[0].getarglist(), short_preamble[1:-1],
+            short_preamble[-1].getarglist(), self.optimizer.patchguardop)
         self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
+                                  args=args + extra,
                                   descr=jitcelltoken.target_tokens[0]))
 
     def filter_extra_jump_args(self, label_args, jump_args):
@@ -153,22 +167,24 @@
             d[arg] = None
         return new_label_args, new_jump_args
 
-    def inline_short_preamble(self, jump_args):
-        sb = self.short_preamble_producer
-        assert len(sb.short_inputargs) == len(jump_args)
+    def inline_short_preamble(self, jump_args, short_inputargs, short_ops,
+                              short_jump_op, patchguardop):
+        self._check_no_forwarding([short_inputargs, short_ops], False)
+        assert len(short_inputargs) == len(jump_args)
         for i in range(len(jump_args)):
-            sb.short_inputargs[i].set_forwarded(None)
-            self.make_equal_to(sb.short_inputargs[i], jump_args[i])
-        patchguardop = self.optimizer.patchguardop
-        for op in sb.short:
+            short_inputargs[i].set_forwarded(None)
+            self.make_equal_to(short_inputargs[i], jump_args[i])
+        for op in short_ops:
             if op.is_guard():
                 op = self.replace_op_with(op, op.getopnum())
                 op.rd_snapshot = patchguardop.rd_snapshot
                 op.rd_frame_info_list = patchguardop.rd_frame_info_list
             self.optimizer.send_extra_operation(op)
         res = [self.optimizer.get_box_replacement(op) for op in
-                sb.short_preamble_jump]
-        for op in sb.short_inputargs:
+                short_jump_op]
+        for op in short_inputargs:
+            op.set_forwarded(None)
+        for op in short_ops:
             op.set_forwarded(None)
         return res
 

From noreply at buildbot.pypy.org  Wed Aug 19 14:03:27 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 19 Aug 2015 14:03:27 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: further cut down the search space
 to move guards to an earlier position,
 at the same time it seems that I have made it weaker. some guards are not
 moved anymore (need investigation)
Message-ID: <20150819120327.1DFA31C0F47@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79052:8fa916fbce20
Date: 2015-08-19 13:19 +0200
http://bitbucket.org/pypy/pypy/changeset/8fa916fbce20/

Log:	further cut down the search space to move guards to an earlier
	position, at the same time it seems that I have made it weaker. some
	guards are not moved anymore (need investigation)

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -674,6 +674,9 @@
         cloned.copy_all_attributes_from(self)
         return cloned
 
+    def exits_early(self):
+        return True
+
 class ResumeGuardNonnullDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_NONNULL
 
diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -4,6 +4,7 @@
 from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method
 from rpython.jit.metainterp.resoperation import (rop, GuardResOp, ResOperation)
 from rpython.jit.metainterp.resume import Snapshot
+from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (BoxPtr, ConstPtr, ConstInt, BoxInt,
     Box, Const, BoxFloat, AbstractValue)
@@ -61,8 +62,13 @@
             count -= 1
         while i < count: 
             op = self.path[i].getoperation()
-            if op.getopnum() != rop.GUARD_EARLY_EXIT and not op.is_always_pure():
+            if not op.is_always_pure():
                 return False
+            if op.is_guard():
+                descr = op.getdescr()
+                assert isinstance(descr, ResumeGuardDescr)
+                if not descr or not descr.exits_early():
+                    return False
             i += 1
         return True
 
@@ -286,9 +292,8 @@
             else:
                 iterdir = node.provides()
             if index >= len(iterdir):
-                #if blacklist:
-                    #blacklist_visit[node] = None
-                #    print "blacklisting 1", node, "path", '->'.join([str(p.opidx) for p in path.path])
+                if blacklist:
+                    blacklist_visit[node] = None
                 continue
             else:
                 next_dep = iterdir[index]
@@ -297,7 +302,6 @@
                 if index < len(iterdir):
                     worklist.append((index, node, pathlen))
                 else:
-                    print "blacklisting 2", node, "path", '->'.join([str(p.opidx) for p in path.path])
                     blacklist_visit[node] = None
                 path.cut_off_at(pathlen)
                 path.walk(next_node)
@@ -385,9 +389,6 @@
         return self.to == to
     def points_at(self, at):
         return self.at == at
-    def i_points_at(self, idx):
-        # REM
-        return self.at.opidx == idx
 
     def add_dependency(self, at, to, arg):
         self.args.append((at,arg))
@@ -576,17 +577,16 @@
         # pass 2 correct guard dependencies
         for guard_node in self.guards:
             self.build_guard_dependencies(guard_node, tracker)
-        # pass 3 find schedulable nodes
-        jump_node = self.nodes[jump_pos]
-        label_node = self.nodes[label_pos]
+
+    def prepare_for_scheduling(self):
+        jump_node = self.nodes[len(self.nodes)-1]
+        jump_node.emitted = True
+        label_node = self.nodes[0]
         for node in self.nodes:
-            if node != jump_node:
-                if node.depends_count() == 0:
-                    self.schedulable_nodes.insert(0, node)
-                # every leaf instruction points to the jump_op. in theory every instruction
-                # points to jump_op. this forces the jump/finish op to be the last operation
-                if node.provides_count() == 0:
-                    node.edge_to(jump_node, None, label='jump')
+            if node.depends_count() == 0:
+                self.schedulable_nodes.insert(0, node)
+        if not we_are_translated():
+            assert self.schedulable_nodes[-1] == label_node
 
     def guard_argument_protection(self, guard_node, tracker):
         """ the parameters the guard protects are an indicator for
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -98,6 +98,11 @@
 
             raise AssertionError("schedule failed cannot continue. possible reason: cycle")
 
+        jump_node = self.graph.nodes[-1]
+        assert jump_node.getopnum() == rop.JUMP
+        self.sched_data.unpack_from_vector(jump_node.getoperation(), self)
+        oplist.append(jump_node.getoperation())
+
 def vectorbox_outof_box(box, count=-1, size=-1, type='-'):
     if box.type not in (FLOAT, INT):
         raise AssertionError("cannot create vector box of type %s" % (box.type))
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -97,6 +97,7 @@
             if cycle is not None:
                 print "CYCLE found %s" % cycle
             self.show_dot_graph(opt.dependency_graph, "early_exit_" + self.test_name)
+            assert cycle is None
             opt.schedule(False)
         opt.unroll_loop_iterations(loop, unroll_factor)
         opt.loop.operations = opt.get_newoperations()
@@ -1348,7 +1349,6 @@
         guard_true(i22, descr=) [p1, p0, p5, p6, p7, p12, p13, i14]
         jump(p0, p1, p5, p6, p7, p12, p13, i31, i15, i16, i17, i18, i19, i20)
         """
-        # schedule 885 -> ptype is non for raw_load?
         opt = self.vectorize(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -469,6 +469,7 @@
         if sched_data is None:
             sched_data = VecScheduleData(self.cpu.vector_register_size,
                                          self.costmodel, self.orig_label_args)
+        self.dependency_graph.prepare_for_scheduling()
         scheduler = Scheduler(self.dependency_graph, sched_data)
         renamer = Renamer()
         #
@@ -515,49 +516,34 @@
         label_node = graph.getnode(0)
         ee_guard_node = graph.getnode(ee_pos)
         guards = graph.guards
-        unique = set()
         for guard_node in guards:
             if guard_node is ee_guard_node:
                 continue
             modify_later = []
             last_prev_node = None
-            for path in guard_node.iterate_paths(ee_guard_node, backwards=True, blacklist=True):
-                p = '->'.join([str(p.opidx) for p in path.path])
-                if p in unique:
-                    assert 0
+            valid = True
+            for prev_dep in guard_node.depends():
+                prev_node = prev_dep.to
+                if prev_dep.is_failarg():
+                    # remove this edge later.
+                    # 1) only because of failing, this dependency exists
+                    # 2) non pure operation points to this guard.
+                    #    but if this guard only depends on pure operations, it can be checked
+                    #    at an earlier position, the non pure op can execute later!
+                    modify_later.append((prev_node, guard_node))
                 else:
-                    unique.add(p)
-                print "PATH:", p
-                if not we_are_translated():
-                    path.check_acyclic()
-                prev_node = path.second()
-                dep = prev_node.depends_on(guard_node)
-                if dep.is_failarg():
-                    # this dependency we are able to break because it is soley
-                    # relevant due to one or multiple fail args
-                    if prev_node is not last_prev_node:
-                        #  ...
-                        #  o  o
-                        #  \ /
-                        #  (a)
-                        #   |
-                        #  (g)
-                        # this graph yields 2 paths from (g), thus (a) is
-                        # remembered and skipped the second time visited
-                        modify_later.append((prev_node, guard_node))
-                        print "  => remove guard -> second"
-                    last_prev_node = prev_node
-                    continue
-                if path.is_always_pure(exclude_first=True, exclude_last=True):
-                    path.set_schedule_priority(10)
-                    if path.last() is ee_guard_node:
-                        modify_later.append((path.last_but_one(), None))
-                        print "  => always pure"
-                else:
-                    # transformation is invalid.
-                    # exit and do not enter else branch!
-                    break
-            else:
+                    for path in prev_node.iterate_paths(ee_guard_node, backwards=True, blacklist=True):
+                        if path.is_always_pure(exclude_first=True, exclude_last=True):
+                            path.set_schedule_priority(10)
+                            if path.last() is ee_guard_node:
+                                modify_later.append((path.last_but_one(), None))
+                        else:
+                            # transformation is invalid.
+                            # exit and do not enter else branch!
+                            valid = False
+                    if not valid:
+                        break
+            if valid:
                 # transformation is valid, modify the graph and execute
                 # this guard earlier
                 for a,b in modify_later:
@@ -568,7 +554,8 @@
                         if last_but_one is ee_guard_node:
                             continue
                         ee_guard_node.remove_edge_to(last_but_one)
-                        label_node.edge_to(last_but_one, label='pullup')
+                        #label_node.edge_to(last_but_one, label='pullup')
+                print "guard", guard_node, "moved earlier"
                 # only the last guard needs a connection
                 guard_node.edge_to(ee_guard_node, label='pullup-last-guard')
                 self.relax_guard_to(guard_node, ee_guard_node)

From noreply at buildbot.pypy.org  Wed Aug 19 14:03:29 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 19 Aug 2015 14:03:29 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: since jump is not scheduled in
 emit_into anymore it needs correct treatment
Message-ID: <20150819120329.23A031C0F47@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79053:6750bbcedc0e
Date: 2015-08-19 13:28 +0200
http://bitbucket.org/pypy/pypy/changeset/6750bbcedc0e/

Log:	since jump is not scheduled in emit_into anymore it needs correct
	treatment

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -99,9 +99,11 @@
             raise AssertionError("schedule failed cannot continue. possible reason: cycle")
 
         jump_node = self.graph.nodes[-1]
-        assert jump_node.getopnum() == rop.JUMP
-        self.sched_data.unpack_from_vector(jump_node.getoperation(), self)
-        oplist.append(jump_node.getoperation())
+        jump_op = jump_node.getoperation()
+        renamer.rename(jump_op)
+        assert jump_op.getopnum() == rop.JUMP
+        self.sched_data.unpack_from_vector(jump_op, self)
+        oplist.append(jump_op)
 
 def vectorbox_outof_box(box, count=-1, size=-1, type='-'):
     if box.type not in (FLOAT, INT):

From noreply at buildbot.pypy.org  Wed Aug 19 14:03:31 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 19 Aug 2015 14:03:31 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: adapted tests to ignore the jump.
 the jump operation is not tracked anymore as a dependency,
 but it is always emitted at the latest position in the trace
Message-ID: <20150819120331.3AE241C0F47@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79054:fc7129745344
Date: 2015-08-19 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/fc7129745344/

Log:	adapted tests to ignore the jump. the jump operation is not tracked
	anymore as a dependency, but it is always emitted at the latest
	position in the trace

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -62,13 +62,15 @@
             count -= 1
         while i < count: 
             op = self.path[i].getoperation()
-            if not op.is_always_pure():
-                return False
             if op.is_guard():
                 descr = op.getdescr()
+                if not descr:
+                    return False
                 assert isinstance(descr, ResumeGuardDescr)
-                if not descr or not descr.exits_early():
+                if not descr.exits_early():
                     return False
+            elif not op.is_always_pure():
+                return False
             i += 1
         return True
 
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -82,6 +82,7 @@
         self.renamer = renamer
         self.oplist = oplist
         self.unpack = unpack
+
         while self.has_more():
             i = self.next_index(self.schedulable_nodes)
             if i >= 0:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -59,7 +59,7 @@
                 dependency = node_a.getedge_to(node_b)
                 if dependency is None and idx_b not in exceptions.setdefault(idx,[]):
                     self.show_dot_graph(graph, self.test_name + '_except')
-                    assert dependency is not None, \
+                    assert dependency is not None or node_b.getopnum() == rop.JUMP, \
                        " it is expected that instruction at index" + \
                        " %s depends on instr on index %s but it does not.\n%s" \
                             % (node_a, node_b, graph)
@@ -168,8 +168,6 @@
         self.assert_dependent(1,2)
         self.assert_dependent(2,3)
         self.assert_dependent(1,3)
-        self.assert_dependent(2,4)
-        self.assert_dependent(3,4)
 
     def test_def_use_jump_use_def(self):
         ops = """
@@ -336,7 +334,6 @@
         jump(p0, i1) # 4:
         """
         self.assert_dependencies(ops, full_check=True)
-        assert self.last_graph.getnode(1).provides_count() == 1
         self.assert_independent(1,2)
         self.assert_independent(1,3) # they modify 2 different cells
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -55,7 +55,7 @@
             for i,op in enumerate(loop.operations):
                 if op.getopnum() == rop.LABEL:
                     op.setdescr(tt)
-        loop.operations = pre + loop.operations
+        loop.operations = pre + filter(lambda op: op.getopnum() != rop.DEBUG_MERGE_POINT, loop.operations)
         if loop.operations[-1].getopnum() == rop.JUMP:
             loop.operations[-1].setdescr(token)
         for op in loop.operations:
@@ -1495,8 +1495,8 @@
         debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #34 FOR_ITER')
         jump(p0, p1, p5, p6, p7, p9, p11, p70)
         """        
-        opt = self.schedule(self.parse_loop(trace))
-        self.debug_print_operations(opt.loop)
+        #opt = self.schedule(self.parse_loop(trace))
+        #self.debug_print_operations(opt.loop)
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -555,7 +555,6 @@
                             continue
                         ee_guard_node.remove_edge_to(last_but_one)
                         #label_node.edge_to(last_but_one, label='pullup')
-                print "guard", guard_node, "moved earlier"
                 # only the last guard needs a connection
                 guard_node.edge_to(ee_guard_node, label='pullup-last-guard')
                 self.relax_guard_to(guard_node, ee_guard_node)

From noreply at buildbot.pypy.org  Wed Aug 19 14:08:10 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 19 Aug 2015 14:08:10 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: typo
Message-ID: <20150819120810.297781C0F47@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79055:ad32f76090e6
Date: 2015-08-19 14:08 +0200
http://bitbucket.org/pypy/pypy/changeset/ad32f76090e6/

Log:	typo

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -799,7 +799,7 @@
         value = parts[1]
         if name == 'enable_opts':
             set_param(driver, 'enable_opts', value)
-        elif name == 'vec_param':
+        elif name == 'vec_params':
             set_param(driver, 'vec_params', value)
         else:
             for name1, _ in unroll_parameters:

From noreply at buildbot.pypy.org  Wed Aug 19 14:54:45 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Wed, 19 Aug 2015 14:54:45 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: why not split the parameters up
 into several different integer ones,
 why not do it this way in the first place?
Message-ID: <20150819125445.AE5A81C1197@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79056:7594920913e7
Date: 2015-08-19 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/7594920913e7/

Log:	why not split the parameters up into several different integer ones,
	why not do it this way in the first place?

diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py
--- a/pypy/module/pypyjit/interp_jit.py
+++ b/pypy/module/pypyjit/interp_jit.py
@@ -137,12 +137,10 @@
     for key, w_value in kwds_w.items():
         if key == 'enable_opts':
             jit.set_param(None, 'enable_opts', space.str_w(w_value))
-        elif key == 'vec_params':
-            jit.set_param(None, 'vec_params', space.str_w(w_value))
         else:
             intval = space.int_w(w_value)
             for name, _ in unroll_parameters:
-                if name == key and name != 'enable_opts' and name != 'vec_params':
+                if name == key and name != 'enable_opts':
                     jit.set_param(None, name, intval)
                     break
             else:
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -71,8 +71,8 @@
                     backendopt=False, trace_limit=sys.maxint, inline=False,
                     loop_longevity=0, retrace_limit=5, function_threshold=4,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
-                    max_unroll_recursion=7, vec=0, vec_params='0:0:50:0.6',
-                    **kwds):
+                    max_unroll_recursion=7, vec=0, vec_all=0, vec_cost=0,
+                    vec_length=50, vec_ratio=6, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -96,7 +96,10 @@
         jd.warmstate.set_param_enable_opts(enable_opts)
         jd.warmstate.set_param_max_unroll_recursion(max_unroll_recursion)
         jd.warmstate.set_param_vec(vec)
-        jd.warmstate.set_param_vec_params(vec_params)
+        jd.warmstate.set_param_vec_all(vec_all)
+        jd.warmstate.set_param_vec_cost(vec_cost)
+        jd.warmstate.set_param_vec_length(vec_length)
+        jd.warmstate.set_param_vec_ratio(vec_ratio)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -303,20 +303,17 @@
     def set_param_vec(self, value):
         self.vec = bool(value)
 
-    def set_param_vec_params(self, value):
-        if NonConstant(False):
-            value = 'blah' # not a constant ''
-        values = value.split(":")
-        self.vec_all = bool(values[0])
-        self.vec_cost = 0
-        if len(values) > 1:
-            self.vec_cost = int(values[1])
-        self.vec_length = 50
-        if len(values) > 2:
-            self.vec_length = int(values[2])
-        self.vec_ratio = 0.60
-        if len(values) > 3:
-            self.vec_ratio = float(values[3])
+    def set_param_vec_all(self, value):
+        self.vec_all = bool(value)
+
+    def set_param_vec_cost(self, value):
+        self.vec_cost = bool(value)
+
+    def set_param_vec_length(self, value):
+        self.vec_length = int(value)
+
+    def set_param_vec_ratio(self, value):
+        self.vec_ratio = value / 10.0
 
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -554,13 +554,11 @@
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
     'vec': 'turn on the vectorization optimization (vecopt). requires sse4.1',
-    'vec_params': 'parameters to the optimization separated by colons. [:[:[:]]]. '
-                  'all = 1: try to vectorize trace loops that occur outside of the numpy library. '
-                  'cost = 0: threshold for which traces to bail. 0 means the costs '
-                  'balance the unpacking, if below the vectorizer bails out. '
-                  'length = 50:  the amount of instructions allowed in "all" traces. '
-                  'ratio = 0.60: the number statements that have vector equivalents divided '
-                  'by the total number of trace instructions.',
+    'vec_all': 'all = 1: try to vectorize trace loops that occur outside of the numpy library.',
+    'vec_cost': 'cost = 0: threshold for which traces to bail. 0 means the costs.',
+    'vec_length': 'length = 50:  the amount of instructions allowed in "all" traces.',
+    'vec_ratio': 'ratio = 6: an integer (0-10 => X / 10) statements that have vector equivalents '
+                 'divided by the total number of trace instructions.',
 }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -577,7 +575,10 @@
               'enable_opts': 'all',
               'max_unroll_recursion': 7,
               'vec': 0,
-              'vec_params': '0:0:50:0.60',
+              'vec_all': 0,
+              'vec_cost': 0,
+              'vec_length': 50,
+              'vec_ratio': 6,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 
@@ -799,11 +800,9 @@
         value = parts[1]
         if name == 'enable_opts':
             set_param(driver, 'enable_opts', value)
-        elif name == 'vec_params':
-            set_param(driver, 'vec_params', value)
         else:
             for name1, _ in unroll_parameters:
-                if name1 == name and name1 != 'vec_params' and name1 != 'enable_opts':
+                if name1 == name and name1 != 'enable_opts':
                     try:
                         set_param(driver, name1, int(value))
                     except ValueError:
@@ -969,8 +968,6 @@
         assert s_name.is_constant()
         if s_name.const == 'enable_opts':
             assert annmodel.SomeString(can_be_None=True).contains(s_value)
-        elif s_name.const == 'vec_params':
-            assert annmodel.SomeString(can_be_None=True).contains(s_value)
         else:
             assert (s_value == annmodel.s_None or
                     annmodel.SomeInteger().contains(s_value))
@@ -986,8 +983,6 @@
         name = hop.args_s[1].const
         if name == 'enable_opts':
             repr = string_repr
-        elif name == 'vec_params':
-            repr = string_repr
         else:
             repr = lltype.Signed
         if (isinstance(hop.args_v[2], Constant) and

From noreply at buildbot.pypy.org  Wed Aug 19 15:10:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 15:10:20 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix a couple more tests
Message-ID: <20150819131020.8E4311C1186@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79057:ecd1d13ff7d7
Date: 2015-08-19 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/ecd1d13ff7d7/

Log:	fix a couple more tests

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -420,8 +420,11 @@
 
 class ArrayStructInfo(ArrayPtrInfo):
     def __init__(self, size, vdescr=None):
+        from rpython.jit.metainterp.optimizeopt import intutils
+
         self.length = size
         lgt = len(vdescr.get_all_fielddescrs())
+        self.lenbound = intutils.ConstIntBound(size)
         self.vdescr = vdescr
         self._items = [None] * (size * lgt)
 
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -484,13 +484,21 @@
             newargs = inputargs
         self.init_inparg_dict_from(newargs)
         self.call_pure_results = call_pure_results
-        for op in ops:
+        if ops[-1].getopnum() in (rop.FINISH, rop.JUMP):
+            last = len(ops) - 1
+            extra_jump = True
+        else:
+            extra_jump = False
+            last = len(ops)
+        for i in range(last):
             self._really_emitted_operation = None
-            self.first_optimization.propagate_forward(op)
+            self.first_optimization.propagate_forward(ops[i])
         #self.loop.operations = self.get_newoperations()
         #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         self.flush()
+        if extra_jump:
+            self.first_optimization.propagate_forward(ops[-1])
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
         return BasicLoopInfo(newargs), self._newoperations
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -568,8 +568,8 @@
         i3 = getfield_gc_i(p3, descr=valuedescr)
         escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
+        p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p1, i1, descr=valuedescr)
-        p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
@@ -584,10 +584,10 @@
         p3sub = getfield_gc_r(p3, descr=nextdescr)
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
+        p2sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2, p2sub, descr=nextdescr)
         p1 = new_with_vtable(descr=nodesize)
-        p2sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p2sub, i1, descr=valuedescr)
-        setfield_gc(p2, p2sub, descr=nextdescr)
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -3040,9 +3040,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p0, p2, descr=nextdescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
@@ -3051,9 +3051,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, 252, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected)
@@ -3080,9 +3080,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p0, p2, descr=nextdescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p2, p3, descr=virtualtokendescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p0, p2, descr=nextdescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [p2, i1]
@@ -3091,9 +3091,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         # the point of this test is that 'i1' should show up in the fail_args
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -145,6 +145,8 @@
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
         target_token = jitcelltoken.target_tokens[0]
         virtual_state = self.get_virtual_state(args)
+        target_virtual_state = target_token.virtual_state
+        
         short_preamble = target_token.short_preamble
         extra = self.inline_short_preamble(args,
             short_preamble[0].getarglist(), short_preamble[1:-1],

From noreply at buildbot.pypy.org  Wed Aug 19 15:10:22 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 15:10:22 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix test_resume
Message-ID: <20150819131022.A6B711C1186@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79058:89142fbe7604
Date: 2015-08-19 15:10 +0200
http://bitbucket.org/pypy/pypy/changeset/89142fbe7604/

Log:	fix test_resume

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -9,7 +9,7 @@
 from rpython.jit.metainterp.history import ConstInt, get_const_ptr_for_string
 from rpython.jit.metainterp import executor, compile, resume
 from rpython.jit.metainterp.resoperation import rop, ResOperation, InputArgInt,\
-     OpHelpers
+     OpHelpers, InputArgRef
 from rpython.rlib.rarithmetic import LONG_BIT
 from rpython.jit.tool.oparser import parse
 
@@ -2483,11 +2483,13 @@
                                  fieldvalue.strip(), strict=False)
                 index += 1
 
-    def check_expanded_fail_descr(self, expectedtext, guard_opnum):
+    def check_expanded_fail_descr(self, expectedtext, guard_opnum, values=None):
         from rpython.jit.metainterp.test.test_resume import ResumeDataFakeReader
         from rpython.jit.metainterp.test.test_resume import MyMetaInterp
         guard_op, = [op for op in self.loop.operations if op.is_guard()]
         fail_args = guard_op.getfailargs()
+        if values is not None:
+            fail_args = values
         fdescr = guard_op.getdescr()
         assert fdescr.guard_opnum == guard_opnum
         reader = ResumeDataFakeReader(fdescr, fail_args,
@@ -2727,11 +2729,12 @@
         jump(p1, i2, i4)
         """
         self.optimize_loop(ops, expected)
-        self.loop.inputargs[0].setref_base(self.nodeaddr)
-        self.check_expanded_fail_descr('''
+        self.check_expanded_fail_descr(
+            '''
             p1.nextdescr = p2
             where p2 is a node_vtable, valuedescr=i2
-            ''', rop.GUARD_TRUE)
+            ''', rop.GUARD_TRUE, values=[InputArgInt(0),
+                                         InputArgRef(self.nodeaddr)])
 
     def test_expand_fail_lazy_setfield_2(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -111,6 +111,7 @@
     
     node = lltype.malloc(NODE)
     node.value = 5
+    node.next = node
     node.parent.typeptr = node_vtable
     nodeaddr = lltype.cast_opaque_ptr(llmemory.GCREF, node)
     #nodebox = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node))
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -1207,16 +1207,16 @@
     modifier.vfieldboxes = {}
 
     vdescr = LLtypeMixin.nodesize2
-    v4 = info.InstancePtrInfo(ConstAddr(LLtypeMixin.node_vtable_adr2,
-                                        LLtypeMixin.cpu), vdescr)
+    ca = ConstAddr(LLtypeMixin.node_vtable_adr2, LLtypeMixin.cpu)
+    v4 = info.InstancePtrInfo(ca, vdescr)
     b4s.set_forwarded(v4)
-    v4.setfield(LLtypeMixin.nextdescr, b2s)
-    v4.setfield(LLtypeMixin.valuedescr, b3s)
-    v4.setfield(LLtypeMixin.otherdescr, b5s)
-    v2 = info.InstancePtrInfo(ConstAddr(LLtypeMixin.node_vtable_adr,
-                                        LLtypeMixin.cpu), LLtypeMixin.nodesize)
-    v2.setfield(LLtypeMixin.nextdescr, b4s)
-    v2.setfield(LLtypeMixin.valuedescr, c1s)
+    v4.setfield(LLtypeMixin.nextdescr, ca, b2s)
+    v4.setfield(LLtypeMixin.valuedescr, ca, b3s)
+    v4.setfield(LLtypeMixin.otherdescr, ca, b5s)
+    ca = ConstAddr(LLtypeMixin.node_vtable_adr, LLtypeMixin.cpu)
+    v2 = info.InstancePtrInfo(ca, LLtypeMixin.nodesize)
+    v2.setfield(LLtypeMixin.nextdescr, b4s, ca)
+    v2.setfield(LLtypeMixin.valuedescr, c1s, ca)
     b2s.set_forwarded(v2)
 
     modifier.register_virtual_fields(b2s, [c1s, None, None, b4s])
@@ -1335,9 +1335,10 @@
     modifier.vfieldboxes = {}
     v2 = info.StructPtrInfo(LLtypeMixin.ssize)
     b2s.set_forwarded(v2)
-    v2.setfield(LLtypeMixin.adescr, c1s)
-    v2.setfield(LLtypeMixin.bdescr, b4s)
-    modifier.register_virtual_fields(b2s, [c1s, b4s])
+    v2.setfield(LLtypeMixin.adescr, b2s, c1s)
+    v2.setfield(LLtypeMixin.abisdescr, b2s, c1s)
+    v2.setfield(LLtypeMixin.bdescr, b2s, b4s)
+    modifier.register_virtual_fields(b2s, [c1s, c1s, b4s])
     liveboxes = []
     modifier._number_virtuals(liveboxes, FakeOptimizer(), 0)
     dump_storage(storage, liveboxes)
@@ -1357,6 +1358,7 @@
     expected = [
         (rop.NEW, [], b2t.getref_base(), LLtypeMixin.ssize),
         (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.adescr),
+        (rop.SETFIELD_GC, [b2t, c1s],  None, LLtypeMixin.abisdescr),
         (rop.SETFIELD_GC, [b2t, b4t], None, LLtypeMixin.bdescr),
         ]
     with CompareableConsts():

From noreply at buildbot.pypy.org  Wed Aug 19 16:02:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 16:02:56 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: hg merge default
Message-ID: <20150819140256.277AE1C1186@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-1.2
Changeset: r2244:6756ac8de92f
Date: 2015-08-19 15:27 +0200
http://bitbucket.org/cffi/cffi/changeset/6756ac8de92f/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -53,7 +53,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/cffi/setuptools_ext.py b/cffi/setuptools_ext.py
--- a/cffi/setuptools_ext.py
+++ b/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,8 @@
 ======================
 
 
-1.2.0
-=====
+v1.2.0
+======
 
 * Out-of-line mode: ``int a[][...];`` can be used to declare a structure
   field or global variable which is, simultaneously, of total length
@@ -56,15 +56,15 @@
 .. __: using.html#alternative-allocators
 
 
-1.1.2
-=====
+v1.1.2
+======
 
 * ``ffi.gc()``: fixed a race condition in multithreaded programs
   introduced in 1.1.1
 
 
-1.1.1
-=====
+v1.1.1
+======
 
 * Out-of-line mode: ``ffi.string()``, ``ffi.buffer()`` and
   ``ffi.getwinerror()`` didn't accept their arguments as keyword
@@ -81,8 +81,8 @@
 .. __: cdef.html#dlopen-note
 
 
-1.1.0
-=====
+v1.1.0
+======
 
 * Out-of-line API mode: we can now declare integer types with
   ``typedef int... foo_t;``.  The exact size and signedness of ``foo_t``
@@ -114,14 +114,14 @@
   in ``build/foo.c``, the .o file would be put in ``build/build/foo.o``.
 
 
-1.0.3
-=====
+v1.0.3
+======
 
 * Same as 1.0.2, apart from doc and test fixes on some platforms.
 
 
-1.0.2
-=====
+v1.0.2
+======
 
 * Variadic C functions (ending in a "..." argument) were not supported
   in the out-of-line ABI mode.  This was a bug---there was even a
@@ -130,8 +130,8 @@
 .. __: overview.html#out-of-line-abi-level
 
 
-1.0.1
-=====
+v1.0.1
+======
 
 * ``ffi.set_source()`` crashed if passed a ``sources=[..]`` argument.
   Fixed by chrippa on pull request #60.
@@ -143,8 +143,8 @@
 * Enums were buggy if you used too many "..." in their definition.
 
 
-1.0.0
-=====
+v1.0.0
+======
 
 * The main news item is out-of-line module generation:
 
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -159,6 +159,35 @@
     assert func.name == 'sin'
     assert func.BType == ', ), , False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -237,6 +266,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -317,15 +317,28 @@
                 import cffi
                 ffi = cffi.FFI()
                 ffi.set_source("pack3.mymod", "/*code would be here*/")
+                ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
             f.write("""if 1:
                 from setuptools import setup
+                from distutils.command.build_ext import build_ext
+                import os
+
+                class TestBuildExt(build_ext):
+                    def pre_run(self, ext, ffi):
+                        assert ffi._hi_there == 42
+                        assert ext.name == "pack3.mymod"
+                        fn = os.path.join(self.build_temp, '..', '..', 'see_me')
+                        open(fn, 'w').close()
+
                 setup(name='example1',
                       version='0.1',
                       packages=['pack3'],
                       package_dir={'': 'src1'},
-                      cffi_modules=["src1/pack3/_build.py:ffi"])
+                      cffi_modules=["src1/pack3/_build.py:ffi"],
+                      cmdclass={'build_ext': TestBuildExt},
+                      )
             """)
 
     @chdir_to_tmp
@@ -334,6 +347,7 @@
         self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None}}})
 
@@ -343,6 +357,7 @@
         self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None,
                                                       'mymod.SO': None}}})

From noreply at buildbot.pypy.org  Wed Aug 19 16:02:58 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 16:02:58 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: Windows test fix
Message-ID: <20150819140258.1DF6D1C1186@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-1.2
Changeset: r2245:70841e56a8fb
Date: 2015-08-19 16:03 +0200
http://bitbucket.org/cffi/cffi/changeset/70841e56a8fb/

Log:	Windows test fix

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -327,9 +327,12 @@
 
                 class TestBuildExt(build_ext):
                     def pre_run(self, ext, ffi):
+                        print '_make_setuptools_api: in pre_run:',
                         assert ffi._hi_there == 42
                         assert ext.name == "pack3.mymod"
-                        fn = os.path.join(self.build_temp, '..', '..', 'see_me')
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print 'creating %r' % (fn,)
                         open(fn, 'w').close()
 
                 setup(name='example1',

From noreply at buildbot.pypy.org  Wed Aug 19 16:12:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 16:12:21 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: Modify the filename
Message-ID: <20150819141221.E76CC1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-1.2
Changeset: r2246:10794ef648ab
Date: 2015-08-19 16:10 +0200
http://bitbucket.org/cffi/cffi/changeset/10794ef648ab/

Log:	Modify the filename

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,7 +51,7 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0-1.tar.gz
 
    - MD5: ...
 

From noreply at buildbot.pypy.org  Wed Aug 19 16:12:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 16:12:23 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: Add md5/sha1
Message-ID: <20150819141223.DDDD41C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-1.2
Changeset: r2247:cf3a2f7e9b29
Date: 2015-08-19 16:12 +0200
http://bitbucket.org/cffi/cffi/changeset/cf3a2f7e9b29/

Log:	Add md5/sha1

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,9 +53,9 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0-1.tar.gz
 
-   - MD5: ...
+   - MD5: 092a40d5e8e8f71742008e28e2b2f803
 
-   - SHA: ...
+   - SHA: 496a9f094b86e09d8a27255f850d7fe3b5115038
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From noreply at buildbot.pypy.org  Wed Aug 19 16:12:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 16:12:25 +0200 (CEST)
Subject: [pypy-commit] cffi default: hg merge release-1.2
Message-ID: <20150819141225.C697D1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2248:060045542075
Date: 2015-08-19 16:12 +0200
http://bitbucket.org/cffi/cffi/changeset/060045542075/

Log:	hg merge release-1.2

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,11 +51,11 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0-1.tar.gz
 
-   - MD5: ...
+   - MD5: 092a40d5e8e8f71742008e28e2b2f803
 
-   - SHA: ...
+   - SHA: 496a9f094b86e09d8a27255f850d7fe3b5115038
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -327,9 +327,12 @@
 
                 class TestBuildExt(build_ext):
                     def pre_run(self, ext, ffi):
+                        print '_make_setuptools_api: in pre_run:',
                         assert ffi._hi_there == 42
                         assert ext.name == "pack3.mymod"
-                        fn = os.path.join(self.build_temp, '..', '..', 'see_me')
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print 'creating %r' % (fn,)
                         open(fn, 'w').close()
 
                 setup(name='example1',

From noreply at buildbot.pypy.org  Wed Aug 19 16:43:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 16:43:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: If translation ends up calling
 build_flow() on something that is a
Message-ID: <20150819144356.3772F1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79059:8cc92a2a2f23
Date: 2015-08-19 16:44 +0200
http://bitbucket.org/pypy/pypy/changeset/8cc92a2a2f23/

Log:	If translation ends up calling build_flow() on something that is a
	built-in function, complain more clearly.

diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -13,6 +13,11 @@
 
 def _assert_rpythonic(func):
     """Raise ValueError if ``func`` is obviously not RPython"""
+    try:
+        func.func_code.co_cellvars
+    except AttributeError:
+        raise ValueError("%r is not RPython: it is likely an unexpected "
+                         "built-in function or type" % (func,))
     if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
         raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
     if func.func_code.co_cellvars:
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1363,6 +1363,15 @@
         simplify_graph(graph)
         assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
 
+    def test_unexpected_builtin_function(self):
+        import itertools
+        e = py.test.raises(ValueError, build_flow, itertools.permutations)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, itertools.tee)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, Exception.__init__)
+        assert ' is not RPython:' in str(e.value)
+
 
 DATA = {'x': 5,
         'y': 6}

From noreply at buildbot.pypy.org  Wed Aug 19 16:55:33 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 16:55:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix test_interpreted since
 subprocess.check_call adds dirname(__file__) not pwd to sys.path
Message-ID: <20150819145533.EE84C1C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79060:6641c2a9442b
Date: 2015-08-19 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/6641c2a9442b/

Log:	fix test_interpreted since subprocess.check_call adds
	dirname(__file__) not pwd to sys.path

diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,6 @@
-import time, os, sys, py
+import time, os, sys
+sys.path += ['.'] # for subprocess in test_interpreted
+import py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
 from rpython.translator.c.test.test_genc import compile

From noreply at buildbot.pypy.org  Wed Aug 19 17:21:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 19 Aug 2015 17:21:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix tests
Message-ID: <20150819152105.F296B1C1186@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79061:ae500e6ab940
Date: 2015-08-19 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ae500e6ab940/

Log:	Fix tests

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,12 @@
             i = 0
             count = 0
             i += 5 * WORD # header
-            assert s[i] == '\x04'
-            i += 1 # marker
-            assert s[i] == '\x04'
-            i += 1 # length
-            i += len('pypy')
+            assert s[i    ] == '\x05'    # MARKER_HEADER
+            assert s[i + 1] == '\x00'    # 0
+            assert s[i + 2] == '\x01'    # VERSION_THREAD_ID
+            assert s[i + 3] == chr(4)    # len('pypy')
+            assert s[i + 4: i + 8] == 'pypy'
+            i += 8
             while i < len(s):
                 if s[i] == '\x03':
                     break
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -42,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
+""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):

From noreply at buildbot.pypy.org  Wed Aug 19 18:23:17 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 18:23:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: make record_exact_class checkable with
	lldebug
Message-ID: <20150819162317.A100A1C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r79062:6775ce8fbaee
Date: 2015-08-19 18:04 +0200
http://bitbucket.org/pypy/pypy/changeset/6775ce8fbaee/

Log:	make record_exact_class checkable with lldebug

diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,6 +1087,16 @@
     """
     assert type(value) is cls
 
+def ll_record_exact_class(ll_value, ll_cls):
+    from rpython.rlib.debug import ll_assert
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.rclass import ll_type
+    ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument")
+    ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments")
+    llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls)
+
+
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_class
 
@@ -1100,12 +1110,10 @@
         from rpython.rtyper import rclass
 
         classrepr = rclass.get_type_repr(hop.rtyper)
-
-        hop.exception_cannot_occur()
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
         v_cls = hop.inputarg(classrepr, arg=1)
-        return hop.genop('jit_record_exact_class', [v_inst, v_cls],
-                         resulttype=lltype.Void)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
     pass

From noreply at buildbot.pypy.org  Wed Aug 19 18:23:19 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 18:23:19 +0200 (CEST)
Subject: [pypy-commit] pypy default: make it an error to combine _immutable_
 = True with quasi-immutable fields
Message-ID: <20150819162319.B73FC1C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r79063:e889968fdd37
Date: 2015-08-19 18:05 +0200
http://bitbucket.org/pypy/pypy/changeset/e889968fdd37/

Log:	make it an error to combine _immutable_ = True with quasi-immutable
	fields

diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -616,26 +616,33 @@
                 while rbase.classdef is not None:
                     immutable_fields.update(rbase.immutable_field_set)
                     rbase = rbase.rbase
-                self._parse_field_list(immutable_fields, accessor)
+                self._parse_field_list(immutable_fields, accessor, hints)
 
-    def _parse_field_list(self, fields, accessor):
+    def _parse_field_list(self, fields, accessor, hints):
         ranking = {}
         for name in fields:
+            quasi = False
             if name.endswith('?[*]'):   # a quasi-immutable field pointing to
                 name = name[:-4]        # an immutable array
                 rank = IR_QUASIIMMUTABLE_ARRAY
+                quasi = True
             elif name.endswith('[*]'):    # for virtualizables' lists
                 name = name[:-3]
                 rank = IR_IMMUTABLE_ARRAY
             elif name.endswith('?'):    # a quasi-immutable field
                 name = name[:-1]
                 rank = IR_QUASIIMMUTABLE
+                quasi = True
             else:                       # a regular immutable/green field
                 rank = IR_IMMUTABLE
             try:
                 mangled_name, r = self._get_field(name)
             except KeyError:
                 continue
+            if quasi and hints.get("immutable"):
+                raise TyperError(
+                    "can't have _immutable_ = True and a quasi-immutable field "
+                    "%s in class %s" % (name, self.classdef))
             ranking[mangled_name] = rank
         accessor.initialize(self.object_type, ranking)
         return ranking
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -943,6 +943,19 @@
                 found.append(op.args[1].value)
         assert found == ['mutate_a', 'mutate_a', 'mutate_b']
 
+    def test_quasi_immutable_clashes_with_immutable(self):
+        from rpython.jit.metainterp.typesystem import deref
+        class A(object):
+            _immutable_ = True
+            _immutable_fields_ = ['a?']
+        def f():
+            a = A()
+            a.x = 42
+            a.a = 142
+            return A()
+        with py.test.raises(TyperError):
+            self.gengraph(f, [])
+
     def test_quasi_immutable_array(self):
         from rpython.jit.metainterp.typesystem import deref
         class A(object):

From noreply at buildbot.pypy.org  Wed Aug 19 18:25:33 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 18:25:33 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: argh! _immutable_ overrides
	quasi-immutability
Message-ID: <20150819162533.B40EC1C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79064:f34e31af6e4f
Date: 2015-08-19 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/f34e31af6e4f/

Log:	argh! _immutable_ overrides quasi-immutability

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -55,9 +55,11 @@
 
 class PyCode(eval.Code):
     "CPython-style code objects."
-    _immutable_ = True
-    _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]",
-                          "co_freevars[*]", "co_cellvars[*]",
+    _immutable_fields_ = ["_signature", "co_argcount", "co_cellvars[*]",
+                          "co_code", "co_consts_w[*]", "co_filename",
+                          "co_firstlineno", "co_flags", "co_freevars[*]",
+                          "co_lnotab", "co_names_w[*]", "co_nlocals",
+                          "co_stacksize", "co_varnames[*]",
                           "_args_as_cellvars[*]", "_known_types_version?"]
 
     def __init__(self, space,  argcount, nlocals, stacksize, flags,

From noreply at buildbot.pypy.org  Wed Aug 19 18:25:36 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 18:25:36 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: merge default
Message-ID: <20150819162536.5BA3C1C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79065:bbcbd47d4cec
Date: 2015-08-19 18:25 +0200
http://bitbucket.org/pypy/pypy/changeset/bbcbd47d4cec/

Log:	merge default

diff too long, truncating to 2000 out of 2204 lines

diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("= 2.6.1,
+   you get a new globals dictionary for every call (but then, all globals
+   dictionaries are all kept alive forever, in ``sys._pypy_execute_source``).
 
 .. function:: int pypy_execute_source_ptr(char* source, void* ptr);
 
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -128,13 +128,7 @@
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
-        after = rffi.aroundstate.after
-        if after: after()
-        source = rffi.charp2str(ll_source)
-        res = _pypy_execute_source(source)
-        before = rffi.aroundstate.before
-        if before: before()
-        return rffi.cast(rffi.INT, res)
+        return pypy_execute_source_ptr(ll_source, 0)
 
     @entrypoint('main', [rffi.CCHARP, lltype.Signed],
                 c_name='pypy_execute_source_ptr')
@@ -142,9 +136,7 @@
         after = rffi.aroundstate.after
         if after: after()
         source = rffi.charp2str(ll_source)
-        space.setitem(w_globals, space.wrap('c_argument'),
-                      space.wrap(ll_ptr))
-        res = _pypy_execute_source(source)
+        res = _pypy_execute_source(source, ll_ptr)
         before = rffi.aroundstate.before
         if before: before()
         return rffi.cast(rffi.INT, res)
@@ -169,15 +161,21 @@
         before = rffi.aroundstate.before
         if before: before()
 
-    w_globals = space.newdict()
-    space.setitem(w_globals, space.wrap('__builtins__'),
-                  space.builtin_modules['__builtin__'])
-
-    def _pypy_execute_source(source):
+    def _pypy_execute_source(source, c_argument):
         try:
-            compiler = space.createcompiler()
-            stmt = compiler.compile(source, 'c callback', 'exec', 0)
-            stmt.exec_code(space, w_globals, w_globals)
+            w_globals = space.newdict(module=True)
+            space.setitem(w_globals, space.wrap('__builtins__'),
+                          space.builtin_modules['__builtin__'])
+            space.setitem(w_globals, space.wrap('c_argument'),
+                          space.wrap(c_argument))
+            space.appexec([space.wrap(source), w_globals], """(src, glob):
+                import sys
+                stmt = compile(src, 'c callback', 'exec')
+                if not hasattr(sys, '_pypy_execute_source'):
+                    sys._pypy_execute_source = []
+                sys._pypy_execute_source.append(glob)
+                exec stmt in glob
+            """)
         except OperationError, e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,12 @@
             i = 0
             count = 0
             i += 5 * WORD # header
-            assert s[i] == '\x04'
-            i += 1 # marker
-            assert s[i] == '\x04'
-            i += 1 # length
-            i += len('pypy')
+            assert s[i    ] == '\x05'    # MARKER_HEADER
+            assert s[i + 1] == '\x00'    # 0
+            assert s[i + 2] == '\x01'    # VERSION_THREAD_ID
+            assert s[i + 3] == chr(4)    # len('pypy')
+            assert s[i + 4: i + 8] == 'pypy'
+            i += 8
             while i < len(s):
                 if s[i] == '\x03':
                     break
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -42,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
+""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -82,7 +82,13 @@
                 w_index = space.int(w_obj)   # wrapped float -> wrapped int or long
             if w_index is None:
                 raise StructError("cannot convert argument to integer")
-        return getattr(space, meth)(w_index)
+        method = getattr(space, meth)
+        try:
+            return method(w_index)
+        except OperationError as e:
+            if e.match(self.space, self.space.w_OverflowError):
+                raise StructError("argument out of range")
+            raise
 
     def accept_bool_arg(self):
         w_obj = self.accept_obj_arg()
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -428,6 +428,9 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_overflow(self):
+        raises(self.struct.error, self.struct.pack, 'i', 1<<65)
+
 
 class AppTestStructBuffer(object):
     spaceconfig = dict(usemodules=['struct', '__pypy__'])
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -13,6 +13,11 @@
 
 def _assert_rpythonic(func):
     """Raise ValueError if ``func`` is obviously not RPython"""
+    try:
+        func.func_code.co_cellvars
+    except AttributeError:
+        raise ValueError("%r is not RPython: it is likely an unexpected "
+                         "built-in function or type" % (func,))
     if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
         raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
     if func.func_code.co_cellvars:
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1363,6 +1363,15 @@
         simplify_graph(graph)
         assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
 
+    def test_unexpected_builtin_function(self):
+        import itertools
+        e = py.test.raises(ValueError, build_flow, itertools.permutations)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, itertools.tee)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, Exception.__init__)
+        assert ' is not RPython:' in str(e.value)
+
 
 DATA = {'x': 5,
         'y': 6}
diff --git a/rpython/jit/backend/detect_cpu.py b/rpython/jit/backend/detect_cpu.py
--- a/rpython/jit/backend/detect_cpu.py
+++ b/rpython/jit/backend/detect_cpu.py
@@ -63,6 +63,7 @@
             'AMD64': MODEL_X86,    # win64
             'armv7l': MODEL_ARM,
             'armv6l': MODEL_ARM,
+            'arm': MODEL_ARM,      # freebsd
             }.get(mach)
 
     if result is None:
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -6,9 +6,9 @@
 #endif
 
 #ifdef RPYTHON_VMPROF
-RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored);
+RPY_EXTERN void vmprof_ignore_signals(int ignored);
 static void pypy_codemap_invalid_set(int ignored) {
-    rpython_vmprof_ignore_signals(ignored);
+    vmprof_ignore_signals(ignored);
 }
 #else
 static void pypy_codemap_invalid_set(int ignored) {
diff --git a/rpython/jit/backend/tool/viewcode.py b/rpython/jit/backend/tool/viewcode.py
--- a/rpython/jit/backend/tool/viewcode.py
+++ b/rpython/jit/backend/tool/viewcode.py
@@ -17,18 +17,6 @@
 import subprocess
 from bisect import bisect_left
 
-# don't use rpython.tool.udir here to avoid removing old usessions which
-# might still contain interesting executables
-udir = py.path.local.make_numbered_dir(prefix='viewcode-', keep=2)
-tmpfile = str(udir.join('dump.tmp'))
-
-# hack hack
-import rpython.tool
-mod = new.module('rpython.tool.udir')
-mod.udir = udir
-sys.modules['rpython.tool.udir'] = mod
-rpython.tool.udir = mod
-
 # ____________________________________________________________
 # Some support code from Psyco.  There is more over there,
 # I am porting it in a lazy fashion...  See py-utils/xam.py
@@ -438,6 +426,18 @@
 # ____________________________________________________________
 
 if __name__ == '__main__':
+    # don't use rpython.tool.udir here to avoid removing old usessions which
+    # might still contain interesting executables
+    udir = py.path.local.make_numbered_dir(prefix='viewcode-', keep=2)
+    tmpfile = str(udir.join('dump.tmp'))
+
+    # hack hack
+    import rpython.tool
+    mod = new.module('rpython.tool.udir')
+    mod.udir = udir
+    sys.modules['rpython.tool.udir'] = mod
+    rpython.tool.udir = mod
+
     if '--text' in sys.argv:
         sys.argv.remove('--text')
         showgraph = False
@@ -463,3 +463,7 @@
         world.show(showtext=True)
     else:
         world.showtextonly()
+else:
+    from rpython.tool.udir import udir
+    tmpfile = str(udir.join('dump.tmp'))
+    
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,6 +1087,16 @@
     """
     assert value is not None and type(value) is cls
 
+def ll_record_exact_class(ll_value, ll_cls):
+    from rpython.rlib.debug import ll_assert
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.rclass import ll_type
+    ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument")
+    ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments")
+    llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls)
+
+
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_class
 
@@ -1099,12 +1109,10 @@
         from rpython.rtyper import rclass
 
         classrepr = rclass.get_type_repr(hop.rtyper)
-
-        hop.exception_cannot_occur()
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
         v_cls = hop.inputarg(classrepr, arg=1)
-        return hop.genop('jit_record_exact_class', [v_inst, v_cls],
-                         resulttype=lltype.Void)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
     pass
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,24 +40,20 @@
         **eci_kwds))
 
 
-    vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP,
-                                  compilation_info=eci)
-    vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT,
+    vmprof_init = rffi.llexternal("vmprof_init",
+                                  [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+                                  rffi.CCHARP, compilation_info=eci)
+    vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT,
+    vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
                                      compilation_info=eci,
                                      save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf",
-                                       [rffi.CCHARP, rffi.LONG],
-                                       lltype.Void, compilation_info=eci)
-
-    ## vmprof_register_virtual_function = rffi.llexternal(
-    ##     "vmprof_register_virtual_function",
-    ##     [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    ##     compilation_info=eci, _nowrapper=True)
-
-    vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals",
+    vmprof_register_virtual_function = rffi.llexternal(
+                                           "vmprof_register_virtual_function",
+                                           [rffi.CCHARP, rffi.LONG, rffi.INT],
+                                           rffi.INT, compilation_info=eci)
+    vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals",
                                             [rffi.INT], lltype.Void,
                                             compilation_info=eci)
     return CInterface(locals())
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,14 +1,12 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import jit, rgc, rposix
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
 from rpython.rtyper.lltypesystem import rffi
 
-MAX_CODES = 8000 - 255
-MAX_FUNC_NAME = 255
+MAX_FUNC_NAME = 1023
 
 # ____________________________________________________________
 
@@ -34,8 +32,6 @@
 
     def _cleanup_(self):
         self.is_enabled = False
-        self.fileno = -1
-        self._current_codes = None
 
     @specialize.argtype(1)
     def register_code(self, code, full_name_func):
@@ -102,18 +98,13 @@
         assert fileno >= 0
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
-        if not (1e-6 <= interval < 1.0):
-            raise VMProfError("bad value for 'interval'")
-        interval_usec = int(interval * 1000000.0)
 
-        p_error = self.cintf.vmprof_init(fileno)
+        p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
-        self.fileno = fileno
-        self._write_header(interval_usec)
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable(interval_usec)
+        res = self.cintf.vmprof_enable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
@@ -125,9 +116,6 @@
         if not self.is_enabled:
             raise VMProfError("vmprof is not enabled")
         self.is_enabled = False
-        if self._current_codes is not None:
-            self._flush_codes()
-        self.fileno = -1
         res = self.cintf.vmprof_disable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
@@ -136,48 +124,8 @@
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
             "and at most %d characters; got '%s'" % (MAX_FUNC_NAME, name))
-        b = self._current_codes
-        if b is None:
-            b = self._current_codes = StringBuilder()
-        b.append('\x02')
-        _write_long_to_string_builder(uid, b)
-        _write_long_to_string_builder(len(name), b)
-        b.append(name)
-        if b.getlength() >= MAX_CODES:
-            self._flush_codes()
-
-    def _flush_codes(self):
-        buf = self._current_codes.build()
-        self._current_codes = None
-        self.cintf.vmprof_write_buf(buf, len(buf))
-        # NOTE: keep in mind that vmprof_write_buf() can only write
-        # a maximum of 8184 bytes.  This should be guaranteed here because:
-        assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184
-
-    def _write_header(self, interval_usec):
-        b = StringBuilder()
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(3, b)
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(interval_usec, b)
-        _write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
-        buf = b.build()
-        self.cintf.vmprof_write_buf(buf, len(buf))
-
-
-def _write_long_to_string_builder(l, b):
-    b.append(chr(l & 0xff))
-    b.append(chr((l >> 8) & 0xff))
-    b.append(chr((l >> 16) & 0xff))
-    b.append(chr((l >> 24) & 0xff))
-    if sys.maxint > 2147483647:
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
+        if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
+            raise VMProfError("vmprof buffers full!  disk full or too slow")
 
 
 def vmprof_execute_code(name, get_code_fn, result_class=None):
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -1,22 +1,3 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in
- * gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *   Armin Rigo - arigo at tunes.org
- *
- */
-
 #define _GNU_SOURCE 1
 
 
@@ -39,431 +20,4 @@
 #endif
 
 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
-
-
-/************************************************************/
-
-// functions copied from libunwind using dlopen
-
-static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-static int (*unw_step)(unw_cursor_t*) = NULL;
-static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static int profile_file = -1;
-
-
-RPY_EXTERN
-char *rpython_vmprof_init(int fd)
-{
-    if (!unw_get_reg) {
-        void *libhandle;
-
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
-            goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
-            goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
-            goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
-            goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
-            goto error;
-    }
-    if (prepare_concurrent_bufs() < 0)
-        return "out of memory";
-
-    assert(fd >= 0);
-    profile_file = fd;
-    return NULL;
-
- error:
-    return dlerror();
-}
-
-/************************************************************/
-
-/* value: last bit is 1 if signals must be ignored; all other bits
-   are a counter for how many threads are currently in a signal handler */
-static long volatile signal_handler_value = 1;
-
-RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
-{
-    if (!ignored) {
-        __sync_fetch_and_and(&signal_handler_value, ~1L);
-    }
-    else {
-        /* set the last bit, and wait until concurrently-running signal
-           handlers finish */
-        while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
-            usleep(1);
-        }
-    }
-}
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH   \
-    ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-struct prof_stacktrace_s {
-    char padding[sizeof(long) - 1];
-    char marker;
-    long count, depth;
-    void *stack[];
-};
-
-static long profile_interval_usec = 0;
-static char atfork_hook_installed = 0;
-
-
-/* ******************************************************
- * libunwind workaround for process JIT frames correctly
- * ******************************************************
- */
-
-#include "rvmprof_get_custom_offset.h"
-
-typedef struct {
-    void* _unused1;
-    void* _unused2;
-    void* sp;
-    void* ip;
-    void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4];
-} vmprof_hacked_unw_cursor_t;
-
-static int vmprof_unw_step(unw_cursor_t *cp, int first_run)
-{
-    void* ip;
-    void* sp;
-    ptrdiff_t sp_offset;
-    unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip);
-    unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
-    if (!first_run) {
-        // make sure we're pointing to the CALL and not to the first
-        // instruction after. If the callee adjusts the stack for us
-        // it's not safe to be at the instruction after
-        ip -= 1;
-    }
-    sp_offset = vmprof_unw_get_custom_offset(ip, cp);
-
-    if (sp_offset == -1) {
-        // it means that the ip is NOT in JITted code, so we can use the
-        // stardard unw_step
-        return unw_step(cp);
-    }
-    else {
-        // this is a horrible hack to manually walk the stack frame, by
-        // setting the IP and SP in the cursor
-        vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp;
-        void* bp = (void*)sp + sp_offset;
-        cp2->sp = bp;
-        bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext)
-{
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    (void)ret;
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) {
-            // found main loop stack frame
-            void* sp;
-            unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-            void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
-            void **arg_ptr = (void**)arg_addr;
-            /* if (mainloop_get_virtual_ip) {
-               ip = mainloop_get_virtual_ip(*arg_ptr);
-               } else { */
-            ip = *arg_ptr;
-        }
-
-        int first_run = (n == 0);
-        result[n++] = ip;
-        n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0)
-            break;
-    }
-    return n;
-}
-
-
-/* *************************************************************
- * the signal handler
- * *************************************************************
- */
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
-{
-    long val = __sync_fetch_and_add(&signal_handler_value, 2L);
-
-    if ((val & 1) == 0) {
-        int saved_errno = errno;
-        int fd = profile_file;
-        assert(fd >= 0);
-
-        struct profbuf_s *p = reserve_buffer(fd);
-        if (p == NULL) {
-            /* ignore this signal: there are no free buffers right now */
-        }
-        else {
-            int depth;
-            struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
-            st->marker = MARKER_STACKTRACE;
-            st->count = 1;
-            st->stack[0] = GetPC((ucontext_t*)ucontext);
-            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
-            depth++;  // To account for pc value in stack[0];
-            st->depth = depth;
-            p->data_offset = offsetof(struct prof_stacktrace_s, marker);
-            p->data_size = (depth * sizeof(void *) +
-                            sizeof(struct prof_stacktrace_s) -
-                            offsetof(struct prof_stacktrace_s, marker));
-            commit_buffer(fd, p);
-        }
-
-        errno = saved_errno;
-    }
-
-    __sync_sub_and_fetch(&signal_handler_value, 2L);
-}
-
-
-/* *************************************************************
- * the setup and teardown functions
- * *************************************************************
- */
-
-static int install_sigprof_handler(void)
-{
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-        sigaction(SIGPROF, &sa, NULL) == -1)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_handler(void)
-{
-    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
-        return -1;
-    return 0;
-}
-
-static int install_sigprof_timer(void)
-{
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = profile_interval_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (profile_interval_usec > 0) {
-        remove_sigprof_timer();
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (profile_interval_usec > 0) {
-        install_sigprof_timer();
-    }
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
-{
-    assert(profile_file >= 0);
-    assert(interval_usec > 0);
-    profile_interval_usec = interval_usec;
-
-    if (install_pthread_atfork_hooks() == -1)
-        goto error;
-    if (install_sigprof_handler() == -1)
-        goto error;
-    if (install_sigprof_timer() == -1)
-        goto error;
-    rpython_vmprof_ignore_signals(0);
-    return 0;
-
- error:
-    profile_file = -1;
-    profile_interval_usec = 0;
-    return -1;
-}
-
-static int _write_all(const void *buf, size_t bufsize)
-{
-    while (bufsize > 0) {
-        ssize_t count = write(profile_file, buf, bufsize);
-        if (count <= 0)
-            return -1;   /* failed */
-        buf += count;
-        bufsize -= count;
-    }
-    return 0;
-}
-
-static int close_profile(void)
-{
-    char buf[4096];
-    ssize_t size;
-    unsigned char marker = MARKER_TRAILER;
-
-    if (_write_all(&marker, 1) < 0)
-        return -1;
-
-#ifdef __linux__
-    // copy /proc/self/maps to the end of the profile file
-    int srcfd = open("/proc/self/maps", O_RDONLY);
-    if (srcfd < 0)
-        return -1;
-
-    while ((size = read(srcfd, buf, sizeof buf)) > 0) {
-        if (_write_all(buf, size) < 0) {
-            close(srcfd);
-            return -1;
-        }
-    }
-    close(srcfd);
-#else
-    // freebsd and mac
-    sprintf(buf, "procstat -v %d", getpid());
-    FILE *srcf = popen(buf, "r");
-    if (!srcf)
-        return -1;
-
-    while ((size = fread(buf, 1, sizeof buf, src))) {
-        if (_write_all(buf, size) < 0) {
-            pclose(srcf);
-            return -1;
-        }
-    }
-    pclose(srcf);
-#endif
-
-    /* don't close() the file descriptor from here */
-    profile_file = -1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_disable(void)
-{
-    rpython_vmprof_ignore_signals(1);
-    profile_interval_usec = 0;
-
-    if (remove_sigprof_timer() == -1)
-        return -1;
-    if (remove_sigprof_handler() == -1)
-        return -1;
-    if (shutdown_concurrent_bufs(profile_file) < 0)
-        return -1;
-    return close_profile();
-}
-
-RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
-{
-    struct profbuf_s *p;
-
-    while ((p = reserve_buffer(profile_file)) == NULL) {
-        /* spin loop waiting for a buffer to be ready; should almost never
-           be the case */
-        usleep(1);
-    }
-
-    if (size > SINGLE_BUF_SIZE)
-        size = SINGLE_BUF_SIZE;
-    memcpy(p->data, buf, size);
-    p->data_size = size;
-
-    commit_buffer(profile_file, p);
-}
+#include "vmprof_main.h"
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
 
-RPY_EXTERN char *rpython_vmprof_init(int);
-RPY_EXTERN void rpython_vmprof_ignore_signals(int);
-RPY_EXTERN int rpython_vmprof_enable(long);
-RPY_EXTERN int rpython_vmprof_disable(void);
-RPY_EXTERN void rpython_vmprof_write_buf(char *, long);
+RPY_EXTERN char *vmprof_init(int, double, char *);
+RPY_EXTERN void vmprof_ignore_signals(int);
+RPY_EXTERN int vmprof_enable(void);
+RPY_EXTERN int vmprof_disable(void);
+RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
diff --git a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
+++ /dev/null
@@ -1,63 +0,0 @@
-
-#ifdef PYPY_JIT_CODEMAP
-void *pypy_find_codemap_at_addr(long addr, long *start_addr);
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr);
-long pypy_jit_stack_depth_at_loc(long loc);
-#endif
-
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
-    intptr_t ip_l = (intptr_t)ip;
-    return pypy_jit_stack_depth_at_loc(ip_l);
-#else
-    return -1;
-#endif
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-                                             void *ip, int max_depth)
-{
-#ifdef PYPY_JIT_CODEMAP
-    void *codemap;
-    long current_pos = 0;
-    intptr_t id;
-    long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    void *tmp;
-
-    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
-    if (codemap == NULL)
-        // not a jit code at all
-        return n;
-
-    // modify the last entry to point to start address and not the random one
-    // in the middle
-    result[n - 1] = (void*)start_addr;
-    result[n] = (void*)2;
-    n++;
-    start = n;
-    while (n < max_depth) {
-        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
-        if (id == -1)
-            // finish
-            break;
-        if (id == 0)
-            continue; // not main codemap
-        result[n++] = (void *)id;
-    }
-    k = 0;
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k - 1];
-        result[n - k - 1] = tmp;
-        k++;
-    }
-    if (n < max_depth) {
-        result[n++] = (void*)3;
-    }
-#endif
-    return n;
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/rvmprof_mt.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rvmprof_mt.h
+++ /dev/null
@@ -1,210 +0,0 @@
-/* Support for multithreaded write() operations */
-
-#include 
-#include 
-
-/* The idea is that we have MAX_NUM_BUFFERS available, all of size
-   SINGLE_BUF_SIZE.  Threads and signal handlers can ask to reserve a
-   buffer, fill it, and finally "commit" it, at which point its
-   content is written into the profile file.  There is no hard
-   guarantee about the order in which the committed blocks are
-   actually written.  We do this with two constrains:
-
-   - write() calls should not overlap; only one thread can be
-     currently calling it.
-
-   - the code needs to be multithread-safe *and* signal-handler-safe,
-     which means it must be written in a wait-free style: never have
-     spin loops waiting for some lock to be released, from any of
-     the functions that can be called from the signal handler!  The
-     code holding the lock could be running in the same thread,
-     currently interrupted by the signal handler.
-
-   The value of MAX_NUM_BUFFERS is a trade-off between too high
-   (lots of unnecessary memory, lots of checking all of them)
-   and too low (risk that there is none left).
-*/
-#define MAX_NUM_BUFFERS  20
-#define SINGLE_BUF_SIZE  (8192 - 2 * sizeof(unsigned int))
-
-#if defined(__i386__) || defined(__amd64__)
-  static inline void write_fence(void) { asm("" : : : "memory"); }
-#else
-  static inline void write_fence(void) { __sync_synchronize(); }
-#endif
-
-
-#define PROFBUF_UNUSED   0
-#define PROFBUF_FILLING  1
-#define PROFBUF_READY    2
-
-
-struct profbuf_s {
-    unsigned int data_size;
-    unsigned int data_offset;
-    char data[SINGLE_BUF_SIZE];
-};
-
-static char volatile profbuf_state[MAX_NUM_BUFFERS];
-static struct profbuf_s *profbuf_all_buffers = NULL;
-static int volatile profbuf_write_lock = 2;
-static long profbuf_pending_write;
-
-
-static void unprepare_concurrent_bufs(void)
-{
-    if (profbuf_all_buffers != NULL) {
-        munmap(profbuf_all_buffers, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS);
-        profbuf_all_buffers = NULL;
-    }
-}
-
-static int prepare_concurrent_bufs(void)
-{
-    assert(sizeof(struct profbuf_s) == 8192);
-
-    unprepare_concurrent_bufs();
-    profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
-                               PROT_READ | PROT_WRITE,
-                               MAP_PRIVATE | MAP_ANONYMOUS,
-                               -1, 0);
-    if (profbuf_all_buffers == MAP_FAILED) {
-        profbuf_all_buffers = NULL;
-        return -1;
-    }
-    memset((char *)profbuf_state, PROFBUF_UNUSED, sizeof(profbuf_state));
-    profbuf_write_lock = 0;
-    profbuf_pending_write = -1;
-    return 0;
-}
-
-static int _write_single_ready_buffer(int fd, long i)
-{
-    /* Try to write to disk the buffer number 'i'.  This function must
-       only be called while we hold the write lock. */
-    assert(profbuf_write_lock != 0);
-
-    if (profbuf_pending_write >= 0) {
-        /* A partially written buffer is waiting.  We'll write the
-           rest of this buffer now, instead of 'i'. */
-        i = profbuf_pending_write;
-        assert(profbuf_state[i] == PROFBUF_READY);
-    }
-
-    if (profbuf_state[i] != PROFBUF_READY) {
-        /* this used to be a race condition: the buffer was written by a
-           different thread already, nothing to do now */
-        return 0;
-    }
-
-    int err;
-    struct profbuf_s *p = &profbuf_all_buffers[i];
-    ssize_t count = write(fd, p->data + p->data_offset, p->data_size);
-    if (count == p->data_size) {
-        profbuf_state[i] = PROFBUF_UNUSED;
-        profbuf_pending_write = -1;
-    }
-    else {
-        if (count > 0) {
-            p->data_offset += count;
-            p->data_size -= count;
-        }
-        profbuf_pending_write = i;
-        if (count < 0)
-            return -1;
-    }
-    return 0;
-}
-
-static void _write_ready_buffers(int fd)
-{
-    long i;
-    int has_write_lock = 0;
-
-    for (i = 0; i < MAX_NUM_BUFFERS; i++) {
-        if (profbuf_state[i] == PROFBUF_READY) {
-            if (!has_write_lock) {
-                if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 1))
-                    return;   /* can't acquire the write lock, give up */
-                has_write_lock = 1;
-            }
-            if (_write_single_ready_buffer(fd, i) < 0)
-                break;
-        }
-    }
-    if (has_write_lock)
-        profbuf_write_lock = 0;
-}
-
-static struct profbuf_s *reserve_buffer(int fd)
-{
-    /* Tries to enter a region of code that fills one buffer.  If
-       successful, returns the profbuf_s.  It fails only if the
-       concurrent buffers are all busy (extreme multithreaded usage).
-
-       This might call write() to emit the data sitting in
-       previously-prepared buffers.  In case of write() error, the
-       error is ignored but unwritten data stays in the buffers.
-    */
-    long i;
-
-    _write_ready_buffers(fd);
-
-    for (i = 0; i < MAX_NUM_BUFFERS; i++) {
-        if (profbuf_state[i] == PROFBUF_UNUSED &&
-            __sync_bool_compare_and_swap(&profbuf_state[i], PROFBUF_UNUSED,
-                                         PROFBUF_FILLING)) {
-            struct profbuf_s *p = &profbuf_all_buffers[i];
-            p->data_size = 0;
-            p->data_offset = 0;
-            return p;
-        }
-    }
-    /* no unused buffer found */
-    return NULL;
-}
-
-static void commit_buffer(int fd, struct profbuf_s *buf)
-{
-    /* Leaves a region of code that filled 'buf'.
-
-       This might call write() to emit the data now ready.  In case of
-       write() error, the error is ignored but unwritten data stays in
-       the buffers.
-    */
-
-    /* Make sure every thread sees the full content of 'buf' */
-    write_fence();
-
-    /* Then set the 'ready' flag */
-    long i = buf - profbuf_all_buffers;
-    assert(profbuf_state[i] == PROFBUF_FILLING);
-    profbuf_state[i] = PROFBUF_READY;
-
-    if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 1)) {
-        /* can't acquire the write lock, ignore */
-    }
-    else {
-        _write_single_ready_buffer(fd, i);
-        profbuf_write_lock = 0;
-    }
-}
-
-static int shutdown_concurrent_bufs(int fd)
-{
-    /* no signal handler can be running concurrently here, because we
-       already did rpython_vmprof_ignore_signals(1) */
-    assert(profbuf_write_lock == 0);
-    profbuf_write_lock = 2;
-
-    /* last attempt to flush buffers */
-    int i;
-    for (i = 0; i < MAX_NUM_BUFFERS; i++) {
-        while (profbuf_state[i] == PROFBUF_READY) {
-            if (_write_single_ready_buffer(fd, i) < 0)
-                return -1;
-        }
-    }
-    unprepare_concurrent_bufs();
-    return 0;
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
rename from rpython/rlib/rvmprof/src/rvmprof_config.h
rename to rpython/rlib/rvmprof/src/vmprof_config.h
diff --git a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -0,0 +1,120 @@
+
+#ifdef PYPY_JIT_CODEMAP
+void *pypy_find_codemap_at_addr(long addr, long *start_addr);
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr);
+long pypy_jit_stack_depth_at_loc(long loc);
+#endif
+
+
+#ifdef CPYTHON_GET_CUSTOM_OFFSET
+static void *tramp_start, *tramp_end;
+#endif
+
+
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
+
+#if defined(PYPY_JIT_CODEMAP)
+
+    intptr_t ip_l = (intptr_t)ip;
+    return pypy_jit_stack_depth_at_loc(ip_l);
+
+#elif defined(CPYTHON_GET_CUSTOM_OFFSET)
+
+    if (ip >= tramp_start && ip <= tramp_end) {
+        // XXX the return value is wrong for all the places before push and
+        //     after pop, fix
+        void *bp;
+        void *sp;
+
+        /* This is a stage2 trampoline created by hotpatch:
+
+               push   %rbx
+               push   %rbp
+               mov    %rsp,%rbp
+               and    $0xfffffffffffffff0,%rsp   // make sure the stack is aligned
+               movabs $0x7ffff687bb10,%rbx
+               callq  *%rbx
+               leaveq 
+               pop    %rbx
+               retq   
+
+           the stack layout is like this:
+
+               +-----------+                      high addresses
+               | ret addr  |
+               +-----------+
+               | saved rbx |   start of the function frame
+               +-----------+
+               | saved rbp |
+               +-----------+
+               | ........  |   <-- rbp
+               +-----------+                      low addresses
+
+           So, the trampoline frame starts at rbp+16, and the return address,
+           is at rbp+24.  The vmprof API requires us to return the offset of
+           the frame relative to sp, hence we have this weird computation.
+
+           XXX (antocuni): I think we could change the API to return directly
+           the frame address instead of the offset; however, this require a
+           change in the PyPy code too
+        */
+
+        unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
+        unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp);
+        return bp+16+8-sp;
+    }
+    return -1;
+
+#else
+
+    return -1;
+
+#endif
+}
+
+static long vmprof_write_header_for_jit_addr(void **result, long n,
+                                             void *ip, int max_depth)
+{
+#ifdef PYPY_JIT_CODEMAP
+    void *codemap;
+    long current_pos = 0;
+    intptr_t id;
+    long start_addr = 0;
+    intptr_t addr = (intptr_t)ip;
+    int start, k;
+    void *tmp;
+
+    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
+    if (codemap == NULL)
+        // not a jit code at all
+        return n;
+
+    // modify the last entry to point to start address and not the random one
+    // in the middle
+    result[n - 1] = (void*)start_addr;
+    result[n] = (void*)2;
+    n++;
+    start = n;
+    while (n < max_depth) {
+        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
+        if (id == -1)
+            // finish
+            break;
+        if (id == 0)
+            continue; // not main codemap
+        result[n++] = (void *)id;
+    }
+    k = 0;
+    while (k < (n - start) / 2) {
+        tmp = result[start + k];
+        result[start + k] = result[n - k - 1];
+        result[n - k - 1] = tmp;
+        k++;
+    }
+    if (n < max_depth) {
+        result[n++] = (void*)3;
+    }
+#endif
+    return n;
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
rename from rpython/rlib/rvmprof/src/rvmprof_getpc.h
rename to rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/rvmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -44,7 +44,7 @@
 #ifndef BASE_GETPC_H_
 #define BASE_GETPC_H_
 
-#include "rvmprof_config.h"
+#include "vmprof_config.h"
 
 // On many linux systems, we may need _GNU_SOURCE to get access to
 // the defined constants that define the register we want to see (eg
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -0,0 +1,556 @@
+/* VMPROF
+ *
+ * statistical sampling profiler specifically designed to profile programs
+ * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
+ * etc.
+ *
+ * The logic to dump the C stack traces is partly stolen from the code in
+ * gperftools.
+ * The file "getpc.h" has been entirely copied from gperftools.
+ *
+ * Tested only on gcc, linux, x86_64.
+ *
+ * Copyright (C) 2014-2015
+ *   Antonio Cuni - anto.cuni at gmail.com
+ *   Maciej Fijalkowski - fijall at gmail.com
+ *   Armin Rigo - arigo at tunes.org
+ *
+ */
+
+#define _GNU_SOURCE 1
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "vmprof_getpc.h"
+#include "vmprof_unwind.h"
+#include "vmprof_mt.h"
+
+
+/************************************************************/
+
+// functions copied from libunwind using dlopen
+
+static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
+static int (*unw_step)(unw_cursor_t*) = NULL;
+static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
+static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
+
+static int profile_file = -1;
+static long prepare_interval_usec;
+static struct profbuf_s *volatile current_codes;
+static void *(*mainloop_get_virtual_ip)(char *) = 0;
+
+static int opened_profile(char *interp_name);
+static void flush_codes(void);
+
+RPY_EXTERN
+char *vmprof_init(int fd, double interval, char *interp_name)
+{
+    if (interval < 1e-6 || interval >= 1.0)
+        return "bad value for 'interval'";
+    prepare_interval_usec = (int)(interval * 1000000.0);
+
+    if (!unw_get_reg) {
+        void *libhandle;
+
+        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
+            goto error;
+        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
+            goto error;
+        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
+            goto error;
+        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
+            goto error;
+        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
+            goto error;
+    }
+    if (prepare_concurrent_bufs() < 0)
+        return "out of memory";
+
+    assert(fd >= 0);
+    profile_file = fd;
+    if (opened_profile(interp_name) < 0) {
+        profile_file = -1;
+        return strerror(errno);
+    }
+    return NULL;
+
+ error:
+    return dlerror();
+}
+
+/************************************************************/
+
+/* value: last bit is 1 if signals must be ignored; all other bits
+   are a counter for how many threads are currently in a signal handler */
+static long volatile signal_handler_value = 1;
+
+RPY_EXTERN
+void vmprof_ignore_signals(int ignored)
+{
+    if (!ignored) {
+        __sync_fetch_and_and(&signal_handler_value, ~1L);
+    }
+    else {
+        /* set the last bit, and wait until concurrently-running signal
+           handlers finish */
+        while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
+            usleep(1);
+        }
+    }
+}
+
+
+/* *************************************************************
+ * functions to write a profile file compatible with gperftools
+ * *************************************************************
+ */
+
+#define MAX_FUNC_NAME 128
+#define MAX_STACK_DEPTH   \
+    ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
+
+#define MARKER_STACKTRACE '\x01'
+#define MARKER_VIRTUAL_IP '\x02'
+#define MARKER_TRAILER '\x03'
+#define MARKER_INTERP_NAME '\x04'   /* deprecated */
+#define MARKER_HEADER '\x05'
+
+#define VERSION_BASE '\x00'
+#define VERSION_THREAD_ID '\x01'
+
+struct prof_stacktrace_s {
+    char padding[sizeof(long) - 1];
+    char marker;
+    long count, depth;
+    void *stack[];
+};
+
+static long profile_interval_usec = 0;
+static char atfork_hook_installed = 0;
+
+
+/* ******************************************************
+ * libunwind workaround for process JIT frames correctly
+ * ******************************************************
+ */
+
+#include "vmprof_get_custom_offset.h"
+
+typedef struct {
+    void* _unused1;
+    void* _unused2;
+    void* sp;
+    void* ip;
+    void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4];
+} vmprof_hacked_unw_cursor_t;
+
+static int vmprof_unw_step(unw_cursor_t *cp, int first_run)
+{
+    void* ip;
+    void* sp;
+    ptrdiff_t sp_offset;
+    unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip);
+    unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
+    if (!first_run) {
+        // make sure we're pointing to the CALL and not to the first
+        // instruction after. If the callee adjusts the stack for us
+        // it's not safe to be at the instruction after
+        ip -= 1;
+    }
+    sp_offset = vmprof_unw_get_custom_offset(ip, cp);
+
+    if (sp_offset == -1) {
+        // it means that the ip is NOT in JITted code, so we can use the
+        // stardard unw_step
+        return unw_step(cp);
+    }
+    else {
+        // this is a horrible hack to manually walk the stack frame, by
+        // setting the IP and SP in the cursor
+        vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp;
+        void* bp = (void*)sp + sp_offset;
+        cp2->sp = bp;
+        bp -= sizeof(void*);
+        cp2->ip = ((void**)bp)[0];
+        // the ret is on the top of the stack minus WORD
+        return 1;
+    }
+}
+
+
+/* *************************************************************
+ * functions to dump the stack trace
+ * *************************************************************
+ */
+
+static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext)
+{
+    void *ip;
+    int n = 0;
+    unw_cursor_t cursor;
+    unw_context_t uc = *ucontext;
+
+    int ret = unw_init_local(&cursor, &uc);
+    assert(ret >= 0);
+    (void)ret;
+
+    while (n < max_depth) {
+        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
+            break;
+        }
+
+        unw_proc_info_t pip;
+        unw_get_proc_info(&cursor, &pip);
+
+        /* if n==0, it means that the signal handler interrupted us while we
+           were in the trampoline, so we are not executing (yet) the real main
+           loop function; just skip it */
+        if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) {
+            // found main loop stack frame
+            void* sp;
+            unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
+            if (mainloop_get_virtual_ip)
+                ip = mainloop_get_virtual_ip((char *)sp);
+            else
+                ip = *(void **)sp;
+        }
+
+        int first_run = (n == 0);
+        result[n++] = ip;
+        n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
+        if (vmprof_unw_step(&cursor, first_run) <= 0)
+            break;
+    }
+    return n;
+}
+
+static void *get_current_thread_id(void)
+{
+    /* xxx This function is a hack on two fronts:
+
+       - It assumes that pthread_self() is async-signal-safe.  This
+         should be true on Linux.  I hope it is also true elsewhere.
+
+       - It abuses pthread_self() by assuming it just returns an
+         integer.  According to comments in CPython's source code, the
+         platforms where it is not the case are rare nowadays.
+
+       An alternative would be to try to look if the information is
+       available in the ucontext_t in the caller.
+    */
+    return (void *)pthread_self();
+}
+
+
+/* *************************************************************
+ * the signal handler
+ * *************************************************************
+ */
+
+static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
+{
+    long val = __sync_fetch_and_add(&signal_handler_value, 2L);
+
+    if ((val & 1) == 0) {
+        int saved_errno = errno;
+        int fd = profile_file;
+        assert(fd >= 0);
+
+        struct profbuf_s *p = reserve_buffer(fd);
+        if (p == NULL) {
+            /* ignore this signal: there are no free buffers right now */
+        }
+        else {
+            int depth;
+            struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
+            st->marker = MARKER_STACKTRACE;
+            st->count = 1;
+            st->stack[0] = GetPC((ucontext_t*)ucontext);
+            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-2, ucontext);
+            depth++;  // To account for pc value in stack[0];
+            st->depth = depth;
+            st->stack[depth++] = get_current_thread_id();
+            p->data_offset = offsetof(struct prof_stacktrace_s, marker);
+            p->data_size = (depth * sizeof(void *) +
+                            sizeof(struct prof_stacktrace_s) -
+                            offsetof(struct prof_stacktrace_s, marker));
+            commit_buffer(fd, p);
+        }
+
+        errno = saved_errno;
+    }
+
+    __sync_sub_and_fetch(&signal_handler_value, 2L);
+}
+
+
+/* *************************************************************
+ * the setup and teardown functions
+ * *************************************************************
+ */
+
+static int install_sigprof_handler(void)
+{
+    struct sigaction sa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_sigaction = sigprof_handler;
+    sa.sa_flags = SA_RESTART | SA_SIGINFO;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPROF, &sa, NULL) == -1)
+        return -1;
+    return 0;
+}
+
+static int remove_sigprof_handler(void)
+{
+    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
+        return -1;
+    return 0;
+}
+
+static int install_sigprof_timer(void)
+{
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = profile_interval_usec;
+    timer.it_value = timer.it_interval;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
+        return -1;
+    return 0;
+}
+
+static int remove_sigprof_timer(void) {
+    static struct itimerval timer;
+    timer.it_interval.tv_sec = 0;
+    timer.it_interval.tv_usec = 0;
+    timer.it_value.tv_sec = 0;
+    timer.it_value.tv_usec = 0;
+    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
+        return -1;
+    return 0;
+}
+
+static void atfork_disable_timer(void) {
+    if (profile_interval_usec > 0) {
+        remove_sigprof_timer();
+    }
+}
+
+static void atfork_enable_timer(void) {
+    if (profile_interval_usec > 0) {
+        install_sigprof_timer();
+    }
+}
+
+static int install_pthread_atfork_hooks(void) {
+    /* this is needed to prevent the problems described there:
+         - http://code.google.com/p/gperftools/issues/detail?id=278
+         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
+
+        TL;DR: if the RSS of the process is large enough, the clone() syscall
+        will be interrupted by the SIGPROF before it can complete, then
+        retried, interrupted again and so on, in an endless loop.  The
+        solution is to disable the timer around the fork, and re-enable it
+        only inside the parent.
+    */
+    if (atfork_hook_installed)
+        return 0;
+    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
+    if (ret != 0)
+        return -1;
+    atfork_hook_installed = 1;
+    return 0;
+}
+
+RPY_EXTERN
+int vmprof_enable(void)
+{
+    assert(profile_file >= 0);
+    assert(prepare_interval_usec > 0);
+    profile_interval_usec = prepare_interval_usec;
+
+    if (install_pthread_atfork_hooks() == -1)
+        goto error;
+    if (install_sigprof_handler() == -1)
+        goto error;
+    if (install_sigprof_timer() == -1)
+        goto error;
+    vmprof_ignore_signals(0);
+    return 0;
+
+ error:
+    profile_file = -1;
+    profile_interval_usec = 0;
+    return -1;
+}
+
+static int _write_all(const void *buf, size_t bufsize)
+{
+    while (bufsize > 0) {
+        ssize_t count = write(profile_file, buf, bufsize);
+        if (count <= 0)
+            return -1;   /* failed */
+        buf += count;
+        bufsize -= count;
+    }
+    return 0;
+}
+
+static int opened_profile(char *interp_name)
+{
+    struct {
+        long hdr[5];
+        char interp_name[259];
+    } header;
+
+    size_t namelen = strnlen(interp_name, 255);
+    current_codes = NULL;
+
+    header.hdr[0] = 0;
+    header.hdr[1] = 3;
+    header.hdr[2] = 0;
+    header.hdr[3] = prepare_interval_usec;
+    header.hdr[4] = 0;
+    header.interp_name[0] = MARKER_HEADER;
+    header.interp_name[1] = '\x00';
+    header.interp_name[2] = VERSION_THREAD_ID;
+    header.interp_name[3] = namelen;
+    memcpy(&header.interp_name[4], interp_name, namelen);
+    return _write_all(&header, 5 * sizeof(long) + 4 + namelen);
+}
+
+static int close_profile(void)
+{
+    char buf[4096];
+    ssize_t size;
+    unsigned char marker = MARKER_TRAILER;
+
+    if (_write_all(&marker, 1) < 0)
+        return -1;
+
+#ifdef __linux__
+    // copy /proc/self/maps to the end of the profile file
+    int srcfd = open("/proc/self/maps", O_RDONLY);
+    if (srcfd < 0)
+        return -1;
+
+    while ((size = read(srcfd, buf, sizeof buf)) > 0) {
+        if (_write_all(buf, size) < 0) {
+            close(srcfd);
+            return -1;
+        }
+    }
+    close(srcfd);
+#else
+    // freebsd and mac
+    sprintf(buf, "procstat -v %d", getpid());
+    FILE *srcf = popen(buf, "r");
+    if (!srcf)
+        return -1;
+
+    while ((size = fread(buf, 1, sizeof buf, src))) {
+        if (_write_all(buf, size) < 0) {
+            pclose(srcf);
+            return -1;
+        }
+    }
+    pclose(srcf);
+#endif
+
+    /* don't close() the file descriptor from here */
+    profile_file = -1;
+    return 0;
+}
+
+RPY_EXTERN
+int vmprof_disable(void)
+{
+    vmprof_ignore_signals(1);
+    profile_interval_usec = 0;
+
+    if (remove_sigprof_timer() == -1)
+        return -1;
+    if (remove_sigprof_handler() == -1)
+        return -1;
+    flush_codes();
+    if (shutdown_concurrent_bufs(profile_file) < 0)
+        return -1;
+    return close_profile();
+}
+
+RPY_EXTERN
+int vmprof_register_virtual_function(char *code_name, long code_uid,
+                                     int auto_retry)
+{
+    long namelen = strnlen(code_name, 1023);
+    long blocklen = 1 + 2 * sizeof(long) + namelen;
+    struct profbuf_s *p;
+    char *t;
+
+ retry:
+    p = current_codes;
+    if (p != NULL) {
+        if (__sync_bool_compare_and_swap(¤t_codes, p, NULL)) {
+            /* grabbed 'current_codes': we will append the current block
+               to it if it contains enough room */
+            size_t freesize = SINGLE_BUF_SIZE - p->data_size;
+            if (freesize < blocklen) {
+                /* full: flush it */
+                commit_buffer(profile_file, p);
+                p = NULL;
+            }
+        }
+        else {
+            /* compare-and-swap failed, don't try again */
+            p = NULL;
+        }
+    }
+
+    if (p == NULL) {
+        p = reserve_buffer(profile_file);
+        if (p == NULL) {
+            /* can't get a free block; should almost never be the
+               case.  Spin loop if allowed, or return a failure code
+               if not (e.g. we're in a signal handler) */
+            if (auto_retry > 0) {
+                auto_retry--;
+                usleep(1);
+                goto retry;
+            }
+            return -1;
+        }
+    }
+
+    t = p->data + p->data_size;
+    p->data_size += blocklen;
+    assert(p->data_size <= SINGLE_BUF_SIZE);
+    *t++ = MARKER_VIRTUAL_IP;
+    memcpy(t, &code_uid, sizeof(long)); t += sizeof(long);
+    memcpy(t, &namelen, sizeof(long)); t += sizeof(long);
+    memcpy(t, code_name, namelen);
+
+    /* try to reattach 'p' to 'current_codes' */
+    if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) {
+        /* failed, flush it */
+        commit_buffer(profile_file, p);
+    }
+    return 0;
+}
+
+static void flush_codes(void)
+{
+    struct profbuf_s *p = current_codes;
+    if (p != NULL) {
+        current_codes = NULL;
+        commit_buffer(profile_file, p);
+    }
+}
diff --git a/rpython/rlib/rvmprof/src/vmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -0,0 +1,217 @@
+/* Support for multithreaded write() operations */
+
+#include 
+#include 
+
+/* The idea is that we have MAX_NUM_BUFFERS available, all of size
+   SINGLE_BUF_SIZE.  Threads and signal handlers can ask to reserve a
+   buffer, fill it, and finally "commit" it, at which point its
+   content is written into the profile file.  There is no hard
+   guarantee about the order in which the committed blocks are
+   actually written.  We do this with two constrains:
+
+   - write() calls should not overlap; only one thread can be
+     currently calling it.
+
+   - the code needs to be multithread-safe *and* signal-handler-safe,
+     which means it must be written in a wait-free style: never have
+     spin loops waiting for some lock to be released, from any of
+     the functions that can be called from the signal handler!  The
+     code holding the lock could be running in the same thread,
+     currently interrupted by the signal handler.
+
+   The value of MAX_NUM_BUFFERS is a trade-off between too high
+   (lots of unnecessary memory, lots of checking all of them)
+   and too low (risk that there is none left).
+*/
+#define MAX_NUM_BUFFERS  20
+#define SINGLE_BUF_SIZE  (8192 - 2 * sizeof(unsigned int))
+
+#if defined(__i386__) || defined(__amd64__)
+  static inline void write_fence(void) { asm("" : : : "memory"); }
+#else
+  static inline void write_fence(void) { __sync_synchronize(); }
+#endif
+
+
+#define PROFBUF_UNUSED   0
+#define PROFBUF_FILLING  1
+#define PROFBUF_READY    2
+
+
+struct profbuf_s {
+    unsigned int data_size;
+    unsigned int data_offset;
+    char data[SINGLE_BUF_SIZE];
+};
+
+static char volatile profbuf_state[MAX_NUM_BUFFERS];
+static struct profbuf_s *profbuf_all_buffers = NULL;
+static int volatile profbuf_write_lock = 2;
+static long profbuf_pending_write;
+
+
+static void unprepare_concurrent_bufs(void)
+{
+    if (profbuf_all_buffers != NULL) {
+        munmap(profbuf_all_buffers, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS);
+        profbuf_all_buffers = NULL;
+    }
+}
+
+static int prepare_concurrent_bufs(void)
+{
+    assert(sizeof(struct profbuf_s) == 8192);
+
+    unprepare_concurrent_bufs();
+    profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
+                               PROT_READ | PROT_WRITE,
+                               MAP_PRIVATE | MAP_ANONYMOUS,
+                               -1, 0);
+    if (profbuf_all_buffers == MAP_FAILED) {
+        profbuf_all_buffers = NULL;
+        return -1;
+    }
+    memset((char *)profbuf_state, PROFBUF_UNUSED, sizeof(profbuf_state));
+    profbuf_write_lock = 0;
+    profbuf_pending_write = -1;
+    return 0;
+}
+
+static int _write_single_ready_buffer(int fd, long i)
+{
+    /* Try to write to disk the buffer number 'i'.  This function must
+       only be called while we hold the write lock. */
+    assert(profbuf_write_lock != 0);
+
+    if (profbuf_pending_write >= 0) {
+        /* A partially written buffer is waiting.  We'll write the
+           rest of this buffer now, instead of 'i'. */

From noreply at buildbot.pypy.org  Wed Aug 19 18:35:31 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 18:35:31 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: need the same here
Message-ID: <20150819163531.2B6001C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79066:7c0aabb3b2ba
Date: 2015-08-19 18:35 +0200
http://bitbucket.org/pypy/pypy/changeset/7c0aabb3b2ba/

Log:	need the same here

diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py
--- a/pypy/interpreter/eval.py
+++ b/pypy/interpreter/eval.py
@@ -9,7 +9,7 @@
 class Code(W_Root):
     """A code is a compiled version of some source code.
     Abstract base class."""
-    _immutable_ = True
+    _immutable_fields_ = ['fast_natural_arity', 'co_name', 'hidden_applevel']
     hidden_applevel = False
 
     # n >= 0 : arity

From noreply at buildbot.pypy.org  Wed Aug 19 19:01:34 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 19:01:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: bad CF, no cookie
Message-ID: <20150819170134.809F11C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: 
Changeset: r79067:83ebc73d4fcb
Date: 2015-08-19 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/83ebc73d4fcb/

Log:	bad CF, no cookie

diff --git a/rpython/rtyper/rvirtualizable.py b/rpython/rtyper/rvirtualizable.py
--- a/rpython/rtyper/rvirtualizable.py
+++ b/rpython/rtyper/rvirtualizable.py
@@ -38,8 +38,8 @@
             else:
                 self._super()._setup_repr(hints = hints)
             c_vfields = self.classdef.classdesc.classdict['_virtualizable_']
-            self.my_redirected_fields = self._parse_field_list(c_vfields.value,
-                                                               self.accessor)
+            self.my_redirected_fields = self._parse_field_list(
+                c_vfields.value, self.accessor, hints)
         else:
             self._super()._setup_repr()
             # ootype needs my_redirected_fields even for subclass. lltype does

From noreply at buildbot.pypy.org  Wed Aug 19 19:03:01 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Wed, 19 Aug 2015 19:03:01 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: merge default
Message-ID: <20150819170301.138691C1186@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79068:a2798953542b
Date: 2015-08-19 19:03 +0200
http://bitbucket.org/pypy/pypy/changeset/a2798953542b/

Log:	merge default

diff --git a/rpython/rtyper/rvirtualizable.py b/rpython/rtyper/rvirtualizable.py
--- a/rpython/rtyper/rvirtualizable.py
+++ b/rpython/rtyper/rvirtualizable.py
@@ -38,8 +38,8 @@
             else:
                 self._super()._setup_repr(hints = hints)
             c_vfields = self.classdef.classdesc.classdict['_virtualizable_']
-            self.my_redirected_fields = self._parse_field_list(c_vfields.value,
-                                                               self.accessor)
+            self.my_redirected_fields = self._parse_field_list(
+                c_vfields.value, self.accessor, hints)
         else:
             self._super()._setup_repr()
             # ootype needs my_redirected_fields even for subclass. lltype does

From noreply at buildbot.pypy.org  Wed Aug 19 20:21:49 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:21:49 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: merge default into branch
Message-ID: <20150819182149.BD7B01C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79069:d72d3d9ffb97
Date: 2015-08-19 18:11 +0300
http://bitbucket.org/pypy/pypy/changeset/d72d3d9ffb97/

Log:	merge default into branch

diff too long, truncating to 2000 out of 31450 lines

diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("<@,:/\$\*\+\-\.\^\|\)\(\?\}\{\=]"
+_LegalKeyChars  = r"\w\d!#%&'~_`><@,:/\$\*\+\-\.\^\|\)\(\?\}\{\="
+_LegalValueChars = _LegalKeyChars + r"\[\]"
 _CookiePattern = re.compile(
     r"(?x)"                       # This is a Verbose pattern
     r"\s*"                        # Optional whitespace at start of cookie
     r"(?P"                   # Start of group 'key'
-    ""+ _LegalCharsPatt +"+?"     # Any word of at least one letter, nongreedy
+    "["+ _LegalKeyChars +"]+?"     # Any word of at least one letter, nongreedy
     r")"                          # End of group 'key'
     r"("                          # Optional group: there may not be a value.
     r"\s*=\s*"                    # Equal Sign
@@ -542,7 +543,7 @@
     r"|"                            # or
     r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr
     r"|"                            # or
-    ""+ _LegalCharsPatt +"*"        # Any word or empty string
+    "["+ _LegalValueChars +"]*"        # Any word or empty string
     r")"                          # End of group 'val'
     r")?"                         # End of optional value group
     r"\s*"                        # Any number of spaces.
diff --git a/lib-python/2.7/SimpleHTTPServer.py b/lib-python/2.7/SimpleHTTPServer.py
--- a/lib-python/2.7/SimpleHTTPServer.py
+++ b/lib-python/2.7/SimpleHTTPServer.py
@@ -14,6 +14,7 @@
 import posixpath
 import BaseHTTPServer
 import urllib
+import urlparse
 import cgi
 import sys
 import shutil
@@ -68,10 +69,14 @@
         path = self.translate_path(self.path)
         f = None
         if os.path.isdir(path):
-            if not self.path.endswith('/'):
+            parts = urlparse.urlsplit(self.path)
+            if not parts.path.endswith('/'):
                 # redirect browser - doing basically what apache does
                 self.send_response(301)
-                self.send_header("Location", self.path + "/")
+                new_parts = (parts[0], parts[1], parts[2] + '/',
+                             parts[3], parts[4])
+                new_url = urlparse.urlunsplit(new_parts)
+                self.send_header("Location", new_url)
                 self.end_headers()
                 return None
             for index in "index.html", "index.htm":
diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py
--- a/lib-python/2.7/_LWPCookieJar.py
+++ b/lib-python/2.7/_LWPCookieJar.py
@@ -18,7 +18,7 @@
                        iso2time, time2isoz)
 
 def lwp_cookie_str(cookie):
-    """Return string representation of Cookie in an the LWP cookie file format.
+    """Return string representation of Cookie in the LWP cookie file format.
 
     Actually, the format is extended a bit -- see module docstring.
 
diff --git a/lib-python/2.7/_abcoll.py b/lib-python/2.7/_abcoll.py
--- a/lib-python/2.7/_abcoll.py
+++ b/lib-python/2.7/_abcoll.py
@@ -548,23 +548,25 @@
             If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
             In either case, this is followed by: for k, v in F.items(): D[k] = v
         '''
-        if len(args) > 2:
-            raise TypeError("update() takes at most 2 positional "
-                            "arguments ({} given)".format(len(args)))
-        elif not args:
-            raise TypeError("update() takes at least 1 argument (0 given)")
+        if not args:
+            raise TypeError("descriptor 'update' of 'MutableMapping' object "
+                            "needs an argument")
         self = args[0]
-        other = args[1] if len(args) >= 2 else ()
-
-        if isinstance(other, Mapping):
-            for key in other:
-                self[key] = other[key]
-        elif hasattr(other, "keys"):
-            for key in other.keys():
-                self[key] = other[key]
-        else:
-            for key, value in other:
-                self[key] = value
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('update expected at most 1 arguments, got %d' %
+                            len(args))
+        if args:
+            other = args[0]
+            if isinstance(other, Mapping):
+                for key in other:
+                    self[key] = other[key]
+            elif hasattr(other, "keys"):
+                for key in other.keys():
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
         for key, value in kwds.items():
             self[key] = value
 
diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py
--- a/lib-python/2.7/_pyio.py
+++ b/lib-python/2.7/_pyio.py
@@ -25,8 +25,8 @@
 DEFAULT_BUFFER_SIZE = 8 * 1024  # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
-# defined in io.py. We don't use real inheritance though, because we don't
-# want to inherit the C implementations.
+# defined in io.py. We don't use real inheritance though, because we don't want
+# to inherit the C implementations.
 
 
 class BlockingIOError(IOError):
@@ -775,7 +775,7 @@
         clsname = self.__class__.__name__
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.{0}>".format(clsname)
         else:
             return "<_pyio.{0} name={1!r}>".format(clsname, name)
@@ -1216,8 +1216,10 @@
         return self.writer.flush()
 
     def close(self):
-        self.writer.close()
-        self.reader.close()
+        try:
+            self.writer.close()
+        finally:
+            self.reader.close()
 
     def isatty(self):
         return self.reader.isatty() or self.writer.isatty()
@@ -1538,7 +1540,7 @@
     def __repr__(self):
         try:
             name = self.name
-        except AttributeError:
+        except Exception:
             return "<_pyio.TextIOWrapper encoding='{0}'>".format(self.encoding)
         else:
             return "<_pyio.TextIOWrapper name={0!r} encoding='{1}'>".format(
diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py
--- a/lib-python/2.7/_strptime.py
+++ b/lib-python/2.7/_strptime.py
@@ -335,9 +335,9 @@
     # though
     week_of_year = -1
     week_of_year_start = -1
-    # weekday and julian defaulted to -1 so as to signal need to calculate
+    # weekday and julian defaulted to None so as to signal need to calculate
     # values
-    weekday = julian = -1
+    weekday = julian = None
     found_dict = found.groupdict()
     for group_key in found_dict.iterkeys():
         # Directives not explicitly handled below:
@@ -434,14 +434,14 @@
         year = 1900
     # If we know the week of the year and what day of that week, we can figure
     # out the Julian day of the year.
-    if julian == -1 and week_of_year != -1 and weekday != -1:
+    if julian is None and week_of_year != -1 and weekday is not None:
         week_starts_Mon = True if week_of_year_start == 0 else False
         julian = _calc_julian_from_U_or_W(year, week_of_year, weekday,
                                             week_starts_Mon)
     # Cannot pre-calculate datetime_date() since can change in Julian
     # calculation and thus could have different value for the day of the week
     # calculation.
-    if julian == -1:
+    if julian is None:
         # Need to add 1 to result since first day of the year is 1, not 0.
         julian = datetime_date(year, month, day).toordinal() - \
                   datetime_date(year, 1, 1).toordinal() + 1
@@ -451,7 +451,7 @@
         year = datetime_result.year
         month = datetime_result.month
         day = datetime_result.day
-    if weekday == -1:
+    if weekday is None:
         weekday = datetime_date(year, month, day).weekday()
     if leap_year_fix:
         # the caller didn't supply a year but asked for Feb 29th. We couldn't
diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py
--- a/lib-python/2.7/aifc.py
+++ b/lib-python/2.7/aifc.py
@@ -357,10 +357,13 @@
         self._soundpos = 0
 
     def close(self):
-        if self._decomp:
-            self._decomp.CloseDecompressor()
-            self._decomp = None
-        self._file.close()
+        decomp = self._decomp
+        try:
+            if decomp:
+                self._decomp = None
+                decomp.CloseDecompressor()
+        finally:
+            self._file.close()
 
     def tell(self):
         return self._soundpos
diff --git a/lib-python/2.7/binhex.py b/lib-python/2.7/binhex.py
--- a/lib-python/2.7/binhex.py
+++ b/lib-python/2.7/binhex.py
@@ -32,7 +32,8 @@
     pass
 
 # States (what have we written)
-[_DID_HEADER, _DID_DATA, _DID_RSRC] = range(3)
+_DID_HEADER = 0
+_DID_DATA = 1
 
 # Various constants
 REASONABLY_LARGE=32768  # Minimal amount we pass the rle-coder
@@ -235,17 +236,22 @@
         self._write(data)
 
     def close(self):
-        if self.state < _DID_DATA:
-            self.close_data()
-        if self.state != _DID_DATA:
-            raise Error, 'Close at the wrong time'
-        if self.rlen != 0:
-            raise Error, \
-                  "Incorrect resource-datasize, diff=%r" % (self.rlen,)
-        self._writecrc()
-        self.ofp.close()
-        self.state = None
-        del self.ofp
+        if self.state is None:
+            return
+        try:
+            if self.state < _DID_DATA:
+                self.close_data()
+            if self.state != _DID_DATA:
+                raise Error, 'Close at the wrong time'
+            if self.rlen != 0:
+                raise Error, \
+                    "Incorrect resource-datasize, diff=%r" % (self.rlen,)
+            self._writecrc()
+        finally:
+            self.state = None
+            ofp = self.ofp
+            del self.ofp
+            ofp.close()
 
 def binhex(inp, out):
     """(infilename, outfilename) - Create binhex-encoded copy of a file"""
@@ -463,11 +469,15 @@
         return self._read(n)
 
     def close(self):
-        if self.rlen:
-            dummy = self.read_rsrc(self.rlen)
-        self._checkcrc()
-        self.state = _DID_RSRC
-        self.ifp.close()
+        if self.state is None:
+            return
+        try:
+            if self.rlen:
+                dummy = self.read_rsrc(self.rlen)
+            self._checkcrc()
+        finally:
+            self.state = None
+            self.ifp.close()
 
 def hexbin(inp, out):
     """(infilename, outfilename) - Decode binhexed file"""
diff --git a/lib-python/2.7/bsddb/test/test_all.py b/lib-python/2.7/bsddb/test/test_all.py
--- a/lib-python/2.7/bsddb/test/test_all.py
+++ b/lib-python/2.7/bsddb/test/test_all.py
@@ -412,9 +412,6 @@
         def get_dbp(self) :
             return self._db
 
-    import string
-    string.letters=[chr(i) for i in xrange(65,91)]
-
     bsddb._db.DBEnv_orig = bsddb._db.DBEnv
     bsddb._db.DB_orig = bsddb._db.DB
     if bsddb.db.version() <= (4, 3) :
diff --git a/lib-python/2.7/bsddb/test/test_basics.py b/lib-python/2.7/bsddb/test/test_basics.py
--- a/lib-python/2.7/bsddb/test/test_basics.py
+++ b/lib-python/2.7/bsddb/test/test_basics.py
@@ -999,7 +999,7 @@
         for x in "The quick brown fox jumped over the lazy dog".split():
             d2.put(x, self.makeData(x))
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d3.put(x, x*70)
 
         d1.sync()
@@ -1047,7 +1047,7 @@
             if verbose:
                 print rec
             rec = c3.next()
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
 
         c1.close()
diff --git a/lib-python/2.7/bsddb/test/test_dbshelve.py b/lib-python/2.7/bsddb/test/test_dbshelve.py
--- a/lib-python/2.7/bsddb/test/test_dbshelve.py
+++ b/lib-python/2.7/bsddb/test/test_dbshelve.py
@@ -59,7 +59,7 @@
             return bytes(key, "iso8859-1")  # 8 bits
 
     def populateDB(self, d):
-        for x in string.letters:
+        for x in string.ascii_letters:
             d[self.mk('S' + x)] = 10 * x           # add a string
             d[self.mk('I' + x)] = ord(x)           # add an integer
             d[self.mk('L' + x)] = [x] * 10         # add a list
diff --git a/lib-python/2.7/bsddb/test/test_get_none.py b/lib-python/2.7/bsddb/test/test_get_none.py
--- a/lib-python/2.7/bsddb/test/test_get_none.py
+++ b/lib-python/2.7/bsddb/test/test_get_none.py
@@ -26,14 +26,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(1)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         data = d.get('bad key')
         self.assertEqual(data, None)
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         c = d.cursor()
@@ -43,7 +43,7 @@
             rec = c.next()
 
         self.assertEqual(rec, None)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
@@ -54,14 +54,14 @@
         d.open(self.filename, db.DB_BTREE, db.DB_CREATE)
         d.set_get_returns_none(0)
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.put(x, x * 40)
 
         self.assertRaises(db.DBNotFoundError, d.get, 'bad key')
         self.assertRaises(KeyError, d.get, 'bad key')
 
-        data = d.get(string.letters[0])
-        self.assertEqual(data, string.letters[0]*40)
+        data = d.get(string.ascii_letters[0])
+        self.assertEqual(data, string.ascii_letters[0]*40)
 
         count = 0
         exceptionHappened = 0
@@ -77,7 +77,7 @@
 
         self.assertNotEqual(rec, None)
         self.assertTrue(exceptionHappened)
-        self.assertEqual(count, len(string.letters))
+        self.assertEqual(count, len(string.ascii_letters))
 
         c.close()
         d.close()
diff --git a/lib-python/2.7/bsddb/test/test_queue.py b/lib-python/2.7/bsddb/test/test_queue.py
--- a/lib-python/2.7/bsddb/test/test_queue.py
+++ b/lib-python/2.7/bsddb/test/test_queue.py
@@ -10,7 +10,6 @@
 
 #----------------------------------------------------------------------
 
- at unittest.skip("fails on Windows; see issue 22943")
 class SimpleQueueTestCase(unittest.TestCase):
     def setUp(self):
         self.filename = get_new_database_path()
@@ -37,17 +36,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
@@ -108,17 +107,17 @@
             print "before appends" + '-' * 30
             pprint(d.stat())
 
-        for x in string.letters:
+        for x in string.ascii_letters:
             d.append(x * 40)
 
-        self.assertEqual(len(d), len(string.letters))
+        self.assertEqual(len(d), len(string.ascii_letters))
 
         d.put(100, "some more data")
         d.put(101, "and some more ")
         d.put(75,  "out of order")
         d.put(1,   "replacement data")
 
-        self.assertEqual(len(d), len(string.letters)+3)
+        self.assertEqual(len(d), len(string.ascii_letters)+3)
 
         if verbose:
             print "before close" + '-' * 30
diff --git a/lib-python/2.7/bsddb/test/test_recno.py b/lib-python/2.7/bsddb/test/test_recno.py
--- a/lib-python/2.7/bsddb/test/test_recno.py
+++ b/lib-python/2.7/bsddb/test/test_recno.py
@@ -4,12 +4,11 @@
 import os, sys
 import errno
 from pprint import pprint
+import string
 import unittest
 
 from test_all import db, test_support, verbose, get_new_environment_path, get_new_database_path
 
-letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
 
 #----------------------------------------------------------------------
 
@@ -39,7 +38,7 @@
 
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             recno = d.append(x * 60)
             self.assertIsInstance(recno, int)
             self.assertGreaterEqual(recno, 1)
@@ -270,7 +269,7 @@
         d.set_re_pad(45)  # ...test both int and char
         d.open(self.filename, db.DB_RECNO, db.DB_CREATE)
 
-        for x in letters:
+        for x in string.ascii_letters:
             d.append(x * 35)    # These will be padded
 
         d.append('.' * 40)      # this one will be exact
diff --git a/lib-python/2.7/chunk.py b/lib-python/2.7/chunk.py
--- a/lib-python/2.7/chunk.py
+++ b/lib-python/2.7/chunk.py
@@ -85,8 +85,10 @@
 
     def close(self):
         if not self.closed:
-            self.skip()
-            self.closed = True
+            try:
+                self.skip()
+            finally:
+                self.closed = True
 
     def isatty(self):
         if self.closed:
diff --git a/lib-python/2.7/codecs.py b/lib-python/2.7/codecs.py
--- a/lib-python/2.7/codecs.py
+++ b/lib-python/2.7/codecs.py
@@ -20,8 +20,14 @@
            "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",
            "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",
            "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",
+           "CodecInfo", "Codec", "IncrementalEncoder", "IncrementalDecoder",
+           "StreamReader", "StreamWriter",
+           "StreamReaderWriter", "StreamRecoder",
+           "getencoder", "getdecoder", "getincrementalencoder",
+           "getincrementaldecoder", "getreader", "getwriter",
+           "encode", "decode", "iterencode", "iterdecode",
            "strict_errors", "ignore_errors", "replace_errors",
-           "xmlcharrefreplace_errors",
+           "xmlcharrefreplace_errors", "backslashreplace_errors",
            "register_error", "lookup_error"]
 
 ### Constants
@@ -1051,7 +1057,7 @@
         during translation.
 
         One example where this happens is cp875.py which decodes
-        multiple character to \u001a.
+        multiple character to \\u001a.
 
     """
     m = {}
diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py
--- a/lib-python/2.7/collections.py
+++ b/lib-python/2.7/collections.py
@@ -330,7 +330,7 @@
     #   http://code.activestate.com/recipes/259174/
     #   Knuth, TAOCP Vol. II section 4.6.3
 
-    def __init__(self, iterable=None, **kwds):
+    def __init__(*args, **kwds):
         '''Create a new, empty Counter object.  And if given, count elements
         from an input iterable.  Or, initialize the count from another mapping
         of elements to their counts.
@@ -341,8 +341,15 @@
         >>> c = Counter(a=4, b=2)                   # a new counter from keyword args
 
         '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
         super(Counter, self).__init__()
-        self.update(iterable, **kwds)
+        self.update(*args, **kwds)
 
     def __missing__(self, key):
         'The count of elements not in the Counter is zero.'
@@ -393,7 +400,7 @@
         raise NotImplementedError(
             'Counter.fromkeys() is undefined.  Use Counter(iterable) instead.')
 
-    def update(self, iterable=None, **kwds):
+    def update(*args, **kwds):
         '''Like dict.update() but add counts instead of replacing them.
 
         Source can be an iterable, a dictionary, or another Counter instance.
@@ -413,6 +420,14 @@
         # contexts.  Instead, we implement straight-addition.  Both the inputs
         # and outputs are allowed to contain zero and negative counts.
 
+        if not args:
+            raise TypeError("descriptor 'update' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             if isinstance(iterable, Mapping):
                 if self:
@@ -428,7 +443,7 @@
         if kwds:
             self.update(kwds)
 
-    def subtract(self, iterable=None, **kwds):
+    def subtract(*args, **kwds):
         '''Like dict.update() but subtracts counts instead of replacing them.
         Counts can be reduced below zero.  Both the inputs and outputs are
         allowed to contain zero and negative counts.
@@ -444,6 +459,14 @@
         -1
 
         '''
+        if not args:
+            raise TypeError("descriptor 'subtract' of 'Counter' object "
+                            "needs an argument")
+        self = args[0]
+        args = args[1:]
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        iterable = args[0] if args else None
         if iterable is not None:
             self_get = self.get
             if isinstance(iterable, Mapping):
diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py
--- a/lib-python/2.7/cookielib.py
+++ b/lib-python/2.7/cookielib.py
@@ -464,26 +464,42 @@
     for ns_header in ns_headers:
         pairs = []
         version_set = False
-        for ii, param in enumerate(re.split(r";\s*", ns_header)):
-            param = param.rstrip()
-            if param == "": continue
-            if "=" not in param:
-                k, v = param, None
-            else:
-                k, v = re.split(r"\s*=\s*", param, 1)
-                k = k.lstrip()
+
+        # XXX: The following does not strictly adhere to RFCs in that empty
+        # names and values are legal (the former will only appear once and will
+        # be overwritten if multiple occurrences are present). This is
+        # mostly to deal with backwards compatibility.
+        for ii, param in enumerate(ns_header.split(';')):
+            param = param.strip()
+
+            key, sep, val = param.partition('=')
+            key = key.strip()
+
+            if not key:
+                if ii == 0:
+                    break
+                else:
+                    continue
+
+            # allow for a distinction between present and empty and missing
+            # altogether
+            val = val.strip() if sep else None
+
             if ii != 0:
-                lc = k.lower()
+                lc = key.lower()
                 if lc in known_attrs:
-                    k = lc
-                if k == "version":
+                    key = lc
+
+                if key == "version":
                     # This is an RFC 2109 cookie.
-                    v = _strip_quotes(v)
+                    if val is not None:
+                        val = _strip_quotes(val)
                     version_set = True
-                if k == "expires":
+                elif key == "expires":
                     # convert expires date to seconds since epoch
-                    v = http2time(_strip_quotes(v))  # None if invalid
-            pairs.append((k, v))
+                    if val is not None:
+                        val = http2time(_strip_quotes(val))  # None if invalid
+            pairs.append((key, val))
 
         if pairs:
             if not version_set:
diff --git a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
--- a/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
+++ b/lib-python/2.7/ctypes/macholib/fetch_macholib.bat
@@ -1,1 +1,1 @@
-svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
+svn export --force http://svn.red-bean.com/bob/macholib/trunk/macholib/ .
diff --git a/lib-python/2.7/ctypes/test/test_find.py b/lib-python/2.7/ctypes/test/test_find.py
--- a/lib-python/2.7/ctypes/test/test_find.py
+++ b/lib-python/2.7/ctypes/test/test_find.py
@@ -32,15 +32,24 @@
     def setUp(self):
         self.gl = self.glu = self.gle = None
         if lib_gl:
-            self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            try:
+                self.gl = CDLL(lib_gl, mode=RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_glu:
-            self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            try:
+                self.glu = CDLL(lib_glu, RTLD_GLOBAL)
+            except OSError:
+                pass
         if lib_gle:
             try:
                 self.gle = CDLL(lib_gle)
             except OSError:
                 pass
 
+    def tearDown(self):
+        self.gl = self.glu = self.gle = None
+
     @unittest.skipUnless(lib_gl, 'lib_gl not available')
     def test_gl(self):
         if self.gl:
diff --git a/lib-python/2.7/ctypes/test/test_pickling.py b/lib-python/2.7/ctypes/test/test_pickling.py
--- a/lib-python/2.7/ctypes/test/test_pickling.py
+++ b/lib-python/2.7/ctypes/test/test_pickling.py
@@ -15,9 +15,9 @@
 class Y(X):
     _fields_ = [("str", c_char_p)]
 
-class PickleTest(unittest.TestCase):
+class PickleTest:
     def dumps(self, item):
-        return pickle.dumps(item)
+        return pickle.dumps(item, self.proto)
 
     def loads(self, item):
         return pickle.loads(item)
@@ -72,17 +72,15 @@
 
     @xfail
     def test_wchar(self):
-        pickle.dumps(c_char("x"))
+        self.dumps(c_char(b"x"))
         # Issue 5049
-        pickle.dumps(c_wchar(u"x"))
+        self.dumps(c_wchar(u"x"))
 
-class PickleTest_1(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 1)
-
-class PickleTest_2(PickleTest):
-    def dumps(self, item):
-        return pickle.dumps(item, 2)
+for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+    name = 'PickleTest_%s' % proto
+    globals()[name] = type(name,
+                           (PickleTest, unittest.TestCase),
+                           {'proto': proto})
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/lib-python/2.7/ctypes/test/test_pointers.py b/lib-python/2.7/ctypes/test/test_pointers.py
--- a/lib-python/2.7/ctypes/test/test_pointers.py
+++ b/lib-python/2.7/ctypes/test/test_pointers.py
@@ -7,8 +7,6 @@
                  c_long, c_ulong, c_longlong, c_ulonglong, c_double, c_float]
 python_types = [int, int, int, int, int, long,
                 int, long, long, long, float, float]
-LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
-large_string = 'T' * 2 ** 25
 
 class PointersTestCase(unittest.TestCase):
 
@@ -191,9 +189,11 @@
             self.assertEqual(bool(mth), True)
 
     def test_pointer_type_name(self):
+        LargeNamedType = type('T' * 2 ** 25, (Structure,), {})
         self.assertTrue(POINTER(LargeNamedType))
 
     def test_pointer_type_str_name(self):
+        large_string = 'T' * 2 ** 25
         self.assertTrue(POINTER(large_string))
 
 if __name__ == '__main__':
diff --git a/lib-python/2.7/ctypes/util.py b/lib-python/2.7/ctypes/util.py
--- a/lib-python/2.7/ctypes/util.py
+++ b/lib-python/2.7/ctypes/util.py
@@ -178,7 +178,7 @@
             res = re.findall(expr, data)
             if not res:
                 return _get_soname(_findLib_gcc(name))
-            res.sort(cmp= lambda x,y: cmp(_num_version(x), _num_version(y)))
+            res.sort(key=_num_version)
             return res[-1]
 
     elif sys.platform == "sunos5":
diff --git a/lib-python/2.7/distutils/__init__.py b/lib-python/2.7/distutils/__init__.py
--- a/lib-python/2.7/distutils/__init__.py
+++ b/lib-python/2.7/distutils/__init__.py
@@ -15,5 +15,5 @@
 # Updated automatically by the Python release process.
 #
 #--start constants--
-__version__ = "2.7.9"
+__version__ = "2.7.10"
 #--end constants--
diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py
--- a/lib-python/2.7/distutils/command/check.py
+++ b/lib-python/2.7/distutils/command/check.py
@@ -126,7 +126,7 @@
         """Returns warnings when the provided data doesn't compile."""
         source_path = StringIO()
         parser = Parser()
-        settings = frontend.OptionParser().get_default_values()
+        settings = frontend.OptionParser(components=(Parser,)).get_default_values()
         settings.tab_width = 4
         settings.pep_references = None
         settings.rfc_references = None
@@ -142,8 +142,8 @@
         document.note_source(source_path, -1)
         try:
             parser.parse(data, document)
-        except AttributeError:
-            reporter.messages.append((-1, 'Could not finish the parsing.',
-                                      '', {}))
+        except AttributeError as e:
+            reporter.messages.append(
+                (-1, 'Could not finish the parsing: %s.' % e, '', {}))
 
         return reporter.messages
diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py
--- a/lib-python/2.7/distutils/dir_util.py
+++ b/lib-python/2.7/distutils/dir_util.py
@@ -83,7 +83,7 @@
     """Create all the empty directories under 'base_dir' needed to put 'files'
     there.
 
-    'base_dir' is just the a name of a directory which doesn't necessarily
+    'base_dir' is just the name of a directory which doesn't necessarily
     exist yet; 'files' is a list of filenames to be interpreted relative to
     'base_dir'.  'base_dir' + the directory portion of every file in 'files'
     will be created if it doesn't already exist.  'mode', 'verbose' and
diff --git a/lib-python/2.7/distutils/tests/test_check.py b/lib-python/2.7/distutils/tests/test_check.py
--- a/lib-python/2.7/distutils/tests/test_check.py
+++ b/lib-python/2.7/distutils/tests/test_check.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf8 -*-
 """Tests for distutils.command.check."""
+import textwrap
 import unittest
 from test.test_support import run_unittest
 
@@ -93,6 +94,36 @@
         cmd = self._run(metadata, strict=1, restructuredtext=1)
         self.assertEqual(cmd._warnings, 0)
 
+    @unittest.skipUnless(HAS_DOCUTILS, "won't test without docutils")
+    def test_check_restructuredtext_with_syntax_highlight(self):
+        # Don't fail if there is a `code` or `code-block` directive
+
+        example_rst_docs = []
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code:: python
+
+                def foo():
+                    pass
+            """))
+        example_rst_docs.append(textwrap.dedent("""\
+            Here's some code:
+
+            .. code-block:: python
+
+                def foo():
+                    pass
+            """))
+
+        for rest_with_code in example_rst_docs:
+            pkg_info, dist = self.create_dist(long_description=rest_with_code)
+            cmd = check(dist)
+            cmd.check_restructuredtext()
+            self.assertEqual(cmd._warnings, 0)
+            msgs = cmd._check_rst_data(rest_with_code)
+            self.assertEqual(len(msgs), 0)
+
     def test_check_all(self):
 
         metadata = {'url': 'xxx', 'author': 'xxx'}
diff --git a/lib-python/2.7/distutils/text_file.py b/lib-python/2.7/distutils/text_file.py
--- a/lib-python/2.7/distutils/text_file.py
+++ b/lib-python/2.7/distutils/text_file.py
@@ -124,11 +124,11 @@
     def close (self):
         """Close the current file and forget everything we know about it
            (filename, current line number)."""
-
-        self.file.close ()
+        file = self.file
         self.file = None
         self.filename = None
         self.current_line = None
+        file.close()
 
 
     def gen_error (self, msg, line=None):
diff --git a/lib-python/2.7/dumbdbm.py b/lib-python/2.7/dumbdbm.py
--- a/lib-python/2.7/dumbdbm.py
+++ b/lib-python/2.7/dumbdbm.py
@@ -21,6 +21,7 @@
 
 """
 
+import ast as _ast
 import os as _os
 import __builtin__
 import UserDict
@@ -85,7 +86,7 @@
             with f:
                 for line in f:
                     line = line.rstrip()
-                    key, pos_and_siz_pair = eval(line)
+                    key, pos_and_siz_pair = _ast.literal_eval(line)
                     self._index[key] = pos_and_siz_pair
 
     # Write the index dict to the directory file.  The original directory
@@ -208,8 +209,10 @@
         return len(self._index)
 
     def close(self):
-        self._commit()
-        self._index = self._datfile = self._dirfile = self._bakfile = None
+        try:
+            self._commit()
+        finally:
+            self._index = self._datfile = self._dirfile = self._bakfile = None
 
     __del__ = close
 
diff --git a/lib-python/2.7/encodings/uu_codec.py b/lib-python/2.7/encodings/uu_codec.py
--- a/lib-python/2.7/encodings/uu_codec.py
+++ b/lib-python/2.7/encodings/uu_codec.py
@@ -84,7 +84,7 @@
             data = a2b_uu(s)
         except binascii.Error, v:
             # Workaround for broken uuencoders by /Fredrik Lundh
-            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) / 3
+            nbytes = (((ord(s[0])-32) & 63) * 4 + 5) // 3
             data = a2b_uu(s[:nbytes])
             #sys.stderr.write("Warning: %s\n" % str(v))
         write(data)
diff --git a/lib-python/2.7/ensurepip/__init__.py b/lib-python/2.7/ensurepip/__init__.py
--- a/lib-python/2.7/ensurepip/__init__.py
+++ b/lib-python/2.7/ensurepip/__init__.py
@@ -12,9 +12,9 @@
 __all__ = ["version", "bootstrap"]
 
 
-_SETUPTOOLS_VERSION = "7.0"
+_SETUPTOOLS_VERSION = "15.2"
 
-_PIP_VERSION = "1.5.6"
+_PIP_VERSION = "6.1.1"
 
 # pip currently requires ssl support, so we try to provide a nicer
 # error message when that is missing (http://bugs.python.org/issue19744)
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/pip-1.5.6-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/pip-6.1.1-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..e59694a019051d58b9a378a1adfc9461b8cec9c3
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-15.2-py2.py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..f153ed376684275e08fcfebdb2de8352fb074171
GIT binary patch

[cut]

diff --git a/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl b/lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl
deleted file mode 100644
Binary file lib-python/2.7/ensurepip/_bundled/setuptools-7.0-py2.py3-none-any.whl has changed
diff --git a/lib-python/2.7/fileinput.py b/lib-python/2.7/fileinput.py
--- a/lib-python/2.7/fileinput.py
+++ b/lib-python/2.7/fileinput.py
@@ -233,8 +233,10 @@
         self.close()
 
     def close(self):
-        self.nextfile()
-        self._files = ()
+        try:
+            self.nextfile()
+        finally:
+            self._files = ()
 
     def __iter__(self):
         return self
@@ -270,23 +272,25 @@
 
         output = self._output
         self._output = 0
-        if output:
-            output.close()
+        try:
+            if output:
+                output.close()
+        finally:
+            file = self._file
+            self._file = 0
+            try:
+                if file and not self._isstdin:
+                    file.close()
+            finally:
+                backupfilename = self._backupfilename
+                self._backupfilename = 0
+                if backupfilename and not self._backup:
+                    try: os.unlink(backupfilename)
+                    except OSError: pass
 
-        file = self._file
-        self._file = 0
-        if file and not self._isstdin:
-            file.close()
-
-        backupfilename = self._backupfilename
-        self._backupfilename = 0
-        if backupfilename and not self._backup:
-            try: os.unlink(backupfilename)
-            except OSError: pass
-
-        self._isstdin = False
-        self._buffer = []
-        self._bufindex = 0
+                self._isstdin = False
+                self._buffer = []
+                self._bufindex = 0
 
     def readline(self):
         try:
diff --git a/lib-python/2.7/fnmatch.py b/lib-python/2.7/fnmatch.py
--- a/lib-python/2.7/fnmatch.py
+++ b/lib-python/2.7/fnmatch.py
@@ -47,12 +47,14 @@
     import os,posixpath
     result=[]
     pat=os.path.normcase(pat)
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    match=_cache[pat].match
+        _cache[pat] = re_pat = re.compile(res)
+    match = re_pat.match
     if os.path is posixpath:
         # normcase on posix is NOP. Optimize it away from the loop.
         for name in names:
@@ -71,12 +73,14 @@
     its arguments.
     """
 
-    if not pat in _cache:
+    try:
+        re_pat = _cache[pat]
+    except KeyError:
         res = translate(pat)
         if len(_cache) >= _MAXCACHE:
             _cache.clear()
-        _cache[pat] = re.compile(res)
-    return _cache[pat].match(name) is not None
+        _cache[pat] = re_pat = re.compile(res)
+    return re_pat.match(name) is not None
 
 def translate(pat):
     """Translate a shell PATTERN to a regular expression.
diff --git a/lib-python/2.7/ftplib.py b/lib-python/2.7/ftplib.py
--- a/lib-python/2.7/ftplib.py
+++ b/lib-python/2.7/ftplib.py
@@ -594,11 +594,16 @@
 
     def close(self):
         '''Close the connection without assuming anything about it.'''
-        if self.file is not None:
-            self.file.close()
-        if self.sock is not None:
-            self.sock.close()
-        self.file = self.sock = None
+        try:
+            file = self.file
+            self.file = None
+            if file is not None:
+                file.close()
+        finally:
+            sock = self.sock
+            self.sock = None
+            if sock is not None:
+                sock.close()
 
 try:
     import ssl
@@ -638,12 +643,24 @@
         '221 Goodbye.'
         >>>
         '''
-        ssl_version = ssl.PROTOCOL_TLSv1
+        ssl_version = ssl.PROTOCOL_SSLv23
 
         def __init__(self, host='', user='', passwd='', acct='', keyfile=None,
-                     certfile=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
+                     certfile=None, context=None,
+                     timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None):
+            if context is not None and keyfile is not None:
+                raise ValueError("context and keyfile arguments are mutually "
+                                 "exclusive")
+            if context is not None and certfile is not None:
+                raise ValueError("context and certfile arguments are mutually "
+                                 "exclusive")
             self.keyfile = keyfile
             self.certfile = certfile
+            if context is None:
+                context = ssl._create_stdlib_context(self.ssl_version,
+                                                     certfile=certfile,
+                                                     keyfile=keyfile)
+            self.context = context
             self._prot_p = False
             FTP.__init__(self, host, user, passwd, acct, timeout)
 
@@ -656,12 +673,12 @@
             '''Set up secure control connection by using TLS/SSL.'''
             if isinstance(self.sock, ssl.SSLSocket):
                 raise ValueError("Already using TLS")
-            if self.ssl_version == ssl.PROTOCOL_TLSv1:
+            if self.ssl_version >= ssl.PROTOCOL_SSLv23:
                 resp = self.voidcmd('AUTH TLS')
             else:
                 resp = self.voidcmd('AUTH SSL')
-            self.sock = ssl.wrap_socket(self.sock, self.keyfile, self.certfile,
-                                        ssl_version=self.ssl_version)
+            self.sock = self.context.wrap_socket(self.sock,
+                                                 server_hostname=self.host)
             self.file = self.sock.makefile(mode='rb')
             return resp
 
@@ -692,8 +709,8 @@
         def ntransfercmd(self, cmd, rest=None):
             conn, size = FTP.ntransfercmd(self, cmd, rest)
             if self._prot_p:
-                conn = ssl.wrap_socket(conn, self.keyfile, self.certfile,
-                                       ssl_version=self.ssl_version)
+                conn = self.context.wrap_socket(conn,
+                                                server_hostname=self.host)
             return conn, size
 
         def retrbinary(self, cmd, callback, blocksize=8192, rest=None):
diff --git a/lib-python/2.7/genericpath.py b/lib-python/2.7/genericpath.py
--- a/lib-python/2.7/genericpath.py
+++ b/lib-python/2.7/genericpath.py
@@ -10,6 +10,14 @@
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/lib-python/2.7/gettext.py b/lib-python/2.7/gettext.py
--- a/lib-python/2.7/gettext.py
+++ b/lib-python/2.7/gettext.py
@@ -52,7 +52,9 @@
 
 __all__ = ['NullTranslations', 'GNUTranslations', 'Catalog',
            'find', 'translation', 'install', 'textdomain', 'bindtextdomain',
-           'dgettext', 'dngettext', 'gettext', 'ngettext',
+           'bind_textdomain_codeset',
+           'dgettext', 'dngettext', 'gettext', 'lgettext', 'ldgettext',
+           'ldngettext', 'lngettext', 'ngettext',
            ]
 
 _default_localedir = os.path.join(sys.prefix, 'share', 'locale')
@@ -294,11 +296,12 @@
             # See if we're looking at GNU .mo conventions for metadata
             if mlen == 0:
                 # Catalog description
-                lastk = k = None
+                lastk = None
                 for item in tmsg.splitlines():
                     item = item.strip()
                     if not item:
                         continue
+                    k = v = None
                     if ':' in item:
                         k, v = item.split(':', 1)
                         k = k.strip().lower()
diff --git a/lib-python/2.7/gzip.py b/lib-python/2.7/gzip.py
--- a/lib-python/2.7/gzip.py
+++ b/lib-python/2.7/gzip.py
@@ -238,9 +238,9 @@
             data = data.tobytes()
 
         if len(data) > 0:
-            self.size = self.size + len(data)
+            self.fileobj.write(self.compress.compress(data))
+            self.size += len(data)
             self.crc = zlib.crc32(data, self.crc) & 0xffffffffL
-            self.fileobj.write( self.compress.compress(data) )
             self.offset += len(data)
 
         return len(data)
@@ -369,19 +369,21 @@
         return self.fileobj is None
 
     def close(self):
-        if self.fileobj is None:
+        fileobj = self.fileobj
+        if fileobj is None:
             return
-        if self.mode == WRITE:
-            self.fileobj.write(self.compress.flush())
-            write32u(self.fileobj, self.crc)
-            # self.size may exceed 2GB, or even 4GB
-            write32u(self.fileobj, self.size & 0xffffffffL)
-            self.fileobj = None
-        elif self.mode == READ:
-            self.fileobj = None
-        if self.myfileobj:
-            self.myfileobj.close()
-            self.myfileobj = None
+        self.fileobj = None
+        try:
+            if self.mode == WRITE:
+                fileobj.write(self.compress.flush())
+                write32u(fileobj, self.crc)
+                # self.size may exceed 2GB, or even 4GB
+                write32u(fileobj, self.size & 0xffffffffL)
+        finally:
+            myfileobj = self.myfileobj
+            if myfileobj:
+                self.myfileobj = None
+                myfileobj.close()
 
     def flush(self,zlib_mode=zlib.Z_SYNC_FLUSH):
         self._check_closed()
diff --git a/lib-python/2.7/hashlib.py b/lib-python/2.7/hashlib.py
--- a/lib-python/2.7/hashlib.py
+++ b/lib-python/2.7/hashlib.py
@@ -187,7 +187,7 @@
 
         def prf(msg, inner=inner, outer=outer):
             # PBKDF2_HMAC uses the password as key. We can re-use the same
-            # digest objects and and just update copies to skip initialization.
+            # digest objects and just update copies to skip initialization.
             icpy = inner.copy()
             ocpy = outer.copy()
             icpy.update(msg)
diff --git a/lib-python/2.7/htmlentitydefs.py b/lib-python/2.7/htmlentitydefs.py
--- a/lib-python/2.7/htmlentitydefs.py
+++ b/lib-python/2.7/htmlentitydefs.py
@@ -1,6 +1,6 @@
 """HTML character entity references."""
 
-# maps the HTML entity name to the Unicode codepoint
+# maps the HTML entity name to the Unicode code point
 name2codepoint = {
     'AElig':    0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1
     'Aacute':   0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1
@@ -256,7 +256,7 @@
     'zwnj':     0x200c, # zero width non-joiner, U+200C NEW RFC 2070
 }
 
-# maps the Unicode codepoint to the HTML entity name
+# maps the Unicode code point to the HTML entity name
 codepoint2name = {}
 
 # maps the HTML entity name to the character
diff --git a/lib-python/2.7/httplib.py b/lib-python/2.7/httplib.py
--- a/lib-python/2.7/httplib.py
+++ b/lib-python/2.7/httplib.py
@@ -68,6 +68,7 @@
 
 from array import array
 import os
+import re
 import socket
 from sys import py3kwarning
 from urlparse import urlsplit
@@ -218,6 +219,38 @@
 # maximum amount of headers accepted
 _MAXHEADERS = 100
 
+# Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2)
+#
+# VCHAR          = %x21-7E
+# obs-text       = %x80-FF
+# header-field   = field-name ":" OWS field-value OWS
+# field-name     = token
+# field-value    = *( field-content / obs-fold )
+# field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+# field-vchar    = VCHAR / obs-text
+#
+# obs-fold       = CRLF 1*( SP / HTAB )
+#                ; obsolete line folding
+#                ; see Section 3.2.4
+
+# token          = 1*tchar
+#
+# tchar          = "!" / "#" / "$" / "%" / "&" / "'" / "*"
+#                / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
+#                / DIGIT / ALPHA
+#                ; any VCHAR, except delimiters
+#
+# VCHAR defined in http://tools.ietf.org/html/rfc5234#appendix-B.1
+
+# the patterns for both name and value are more leniant than RFC
+# definitions to allow for backwards compatibility
+_is_legal_header_name = re.compile(r'\A[^:\s][^:\r\n]*\Z').match
+_is_illegal_header_value = re.compile(r'\n(?![ \t])|\r(?![ \t\n])').search
+
+# We always set the Content-Length header for these methods because some
+# servers will otherwise respond with a 411
+_METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'}
+
 
 class HTTPMessage(mimetools.Message):
 
@@ -313,6 +346,11 @@
                 hlist.append(line)
                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
                 continue
+            elif headerseen is not None:
+                # An empty header name. These aren't allowed in HTTP, but it's
+                # probably a benign mistake. Don't add the header, just keep
+                # going.
+                continue
             else:
                 # It's not a header line; throw it back and stop here.
                 if not self.dict:
@@ -522,9 +560,10 @@
         return True
 
     def close(self):
-        if self.fp:
-            self.fp.close()
+        fp = self.fp
+        if fp:
             self.fp = None
+            fp.close()
 
     def isclosed(self):
         # NOTE: it is possible that we will not ever call self.close(). This
@@ -723,7 +762,7 @@
         endpoint passed to set_tunnel. This is done by sending a HTTP CONNECT
         request to the proxy server when the connection is established.
 
-        This method must be called before the HTML connection has been
+        This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers
@@ -797,13 +836,17 @@
 
     def close(self):
         """Close the connection to the HTTP server."""
-        if self.sock:
-            self.sock.close()   # close it manually... there may be other refs
-            self.sock = None
-        if self.__response:
-            self.__response.close()
-            self.__response = None
         self.__state = _CS_IDLE
+        try:
+            sock = self.sock
+            if sock:
+                self.sock = None
+                sock.close()   # close it manually... there may be other refs
+        finally:
+            response = self.__response
+            if response:
+                self.__response = None
+                response.close()
 
     def send(self, data):
         """Send `data' to the server."""
@@ -978,7 +1021,16 @@
         if self.__state != _CS_REQ_STARTED:
             raise CannotSendHeader()
 
-        hdr = '%s: %s' % (header, '\r\n\t'.join([str(v) for v in values]))
+        header = '%s' % header
+        if not _is_legal_header_name(header):
+            raise ValueError('Invalid header name %r' % (header,))
+
+        values = [str(v) for v in values]
+        for one_value in values:
+            if _is_illegal_header_value(one_value):
+                raise ValueError('Invalid header value %r' % (one_value,))
+
+        hdr = '%s: %s' % (header, '\r\n\t'.join(values))
         self._output(hdr)
 
     def endheaders(self, message_body=None):
@@ -1000,19 +1052,25 @@
         """Send a complete request to the server."""
         self._send_request(method, url, body, headers)
 
-    def _set_content_length(self, body):
-        # Set the content-length based on the body.
+    def _set_content_length(self, body, method):
+        # Set the content-length based on the body. If the body is "empty", we
+        # set Content-Length: 0 for methods that expect a body (RFC 7230,
+        # Section 3.3.2). If the body is set for other methods, we set the
+        # header provided we can figure out what the length is.
         thelen = None
-        try:
-            thelen = str(len(body))
-        except TypeError, te:
-            # If this is a file-like object, try to
-            # fstat its file descriptor
+        if body is None and method.upper() in _METHODS_EXPECTING_BODY:
+            thelen = '0'
+        elif body is not None:
             try:
-                thelen = str(os.fstat(body.fileno()).st_size)
-            except (AttributeError, OSError):
-                # Don't send a length if this failed
-                if self.debuglevel > 0: print "Cannot stat!!"
+                thelen = str(len(body))
+            except TypeError:
+                # If this is a file-like object, try to
+                # fstat its file descriptor
+                try:
+                    thelen = str(os.fstat(body.fileno()).st_size)
+                except (AttributeError, OSError):
+                    # Don't send a length if this failed
+                    if self.debuglevel > 0: print "Cannot stat!!"
 
         if thelen is not None:
             self.putheader('Content-Length', thelen)
@@ -1028,8 +1086,8 @@
 
         self.putrequest(method, url, **skips)
 
-        if body is not None and 'content-length' not in header_names:
-            self._set_content_length(body)
+        if 'content-length' not in header_names:
+            self._set_content_length(body, method)
         for hdr, value in headers.iteritems():
             self.putheader(hdr, value)
         self.endheaders(body)
@@ -1072,20 +1130,20 @@
 
         try:
             response.begin()
+            assert response.will_close != _UNKNOWN
+            self.__state = _CS_IDLE
+
+            if response.will_close:
+                # this effectively passes the connection to the response
+                self.close()
+            else:
+                # remember this, so we can tell when it is complete
+                self.__response = response
+
+            return response
         except:
             response.close()
             raise
-        assert response.will_close != _UNKNOWN
-        self.__state = _CS_IDLE
-
-        if response.will_close:
-            # this effectively passes the connection to the response
-            self.close()
-        else:
-            # remember this, so we can tell when it is complete
-            self.__response = response
-
-        return response
 
 
 class HTTP:
@@ -1129,7 +1187,7 @@
         "Accept arguments to set the host/port, since the superclass doesn't."
 
         if host is not None:
-            self._conn._set_hostport(host, port)
+            (self._conn.host, self._conn.port) = self._conn._get_hostport(host, port)
         self._conn.connect()
 
     def getfile(self):
diff --git a/lib-python/2.7/idlelib/CodeContext.py b/lib-python/2.7/idlelib/CodeContext.py
--- a/lib-python/2.7/idlelib/CodeContext.py
+++ b/lib-python/2.7/idlelib/CodeContext.py
@@ -15,8 +15,8 @@
 from sys import maxint as INFINITY
 from idlelib.configHandler import idleConf
 
-BLOCKOPENERS = set(["class", "def", "elif", "else", "except", "finally", "for",
-                    "if", "try", "while", "with"])
+BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for",
+                    "if", "try", "while", "with"}
 UPDATEINTERVAL = 100 # millisec
 FONTUPDATEINTERVAL = 1000 # millisec
 
diff --git a/lib-python/2.7/idlelib/EditorWindow.py b/lib-python/2.7/idlelib/EditorWindow.py
--- a/lib-python/2.7/idlelib/EditorWindow.py
+++ b/lib-python/2.7/idlelib/EditorWindow.py
@@ -469,13 +469,10 @@
         ("format", "F_ormat"),
         ("run", "_Run"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     def createmenubar(self):
         mbar = self.menubar
diff --git a/lib-python/2.7/idlelib/FormatParagraph.py b/lib-python/2.7/idlelib/FormatParagraph.py
--- a/lib-python/2.7/idlelib/FormatParagraph.py
+++ b/lib-python/2.7/idlelib/FormatParagraph.py
@@ -44,9 +44,11 @@
 
         The length limit parameter is for testing with a known value.
         """
-        if limit == None:
+        if limit is None:
+            # The default length limit is that defined by pep8
             limit = idleConf.GetOption(
-                    'main', 'FormatParagraph', 'paragraph', type='int')
+                'extensions', 'FormatParagraph', 'max-width',
+                type='int', default=72)
         text = self.editwin.text
         first, last = self.editwin.get_selection_indices()
         if first and last:
diff --git a/lib-python/2.7/idlelib/PyShell.py b/lib-python/2.7/idlelib/PyShell.py
--- a/lib-python/2.7/idlelib/PyShell.py
+++ b/lib-python/2.7/idlelib/PyShell.py
@@ -871,13 +871,10 @@
         ("edit", "_Edit"),
         ("debug", "_Debug"),
         ("options", "_Options"),
-        ("windows", "_Windows"),
+        ("windows", "_Window"),
         ("help", "_Help"),
     ]
 
-    if sys.platform == "darwin":
-        menu_specs[-2] = ("windows", "_Window")
-
 
     # New classes
     from idlelib.IdleHistory import History
@@ -1350,7 +1347,7 @@
         if type(s) not in (unicode, str, bytearray):
             # See issue #19481
             if isinstance(s, unicode):
-                s = unicode.__getslice__(s, None, None)
+                s = unicode.__getitem__(s, slice(None))
             elif isinstance(s, str):
                 s = str.__str__(s)
             elif isinstance(s, bytearray):
diff --git a/lib-python/2.7/idlelib/SearchEngine.py b/lib-python/2.7/idlelib/SearchEngine.py
--- a/lib-python/2.7/idlelib/SearchEngine.py
+++ b/lib-python/2.7/idlelib/SearchEngine.py
@@ -191,7 +191,7 @@
 
     This is done by searching forwards until there is no match.
     Prog: compiled re object with a search method returning a match.
-    Chars: line of text, without \n.
+    Chars: line of text, without \\n.
     Col: stop index for the search; the limit for match.end().
     '''
     m = prog.search(chars)
diff --git a/lib-python/2.7/idlelib/config-extensions.def b/lib-python/2.7/idlelib/config-extensions.def
--- a/lib-python/2.7/idlelib/config-extensions.def
+++ b/lib-python/2.7/idlelib/config-extensions.def
@@ -66,6 +66,7 @@
 
 [FormatParagraph]
 enable=True
+max-width=72
 [FormatParagraph_cfgBindings]
 format-paragraph=
 
diff --git a/lib-python/2.7/idlelib/config-main.def b/lib-python/2.7/idlelib/config-main.def
--- a/lib-python/2.7/idlelib/config-main.def
+++ b/lib-python/2.7/idlelib/config-main.def
@@ -58,9 +58,6 @@
 font-bold= 0
 encoding= none
 
-[FormatParagraph]
-paragraph=72
-
 [Indent]
 use-spaces= 1
 num-spaces= 4
diff --git a/lib-python/2.7/idlelib/configDialog.py b/lib-python/2.7/idlelib/configDialog.py
--- a/lib-python/2.7/idlelib/configDialog.py
+++ b/lib-python/2.7/idlelib/configDialog.py
@@ -371,7 +371,6 @@
         parent = self.parent
         self.winWidth = StringVar(parent)
         self.winHeight = StringVar(parent)
-        self.paraWidth = StringVar(parent)
         self.startupEdit = IntVar(parent)
         self.autoSave = IntVar(parent)
         self.encoding = StringVar(parent)
@@ -387,7 +386,6 @@
         frameSave = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Autosave Preferences ')
         frameWinSize = Frame(frame, borderwidth=2, relief=GROOVE)
-        frameParaSize = Frame(frame, borderwidth=2, relief=GROOVE)
         frameEncoding = Frame(frame, borderwidth=2, relief=GROOVE)
         frameHelp = LabelFrame(frame, borderwidth=2, relief=GROOVE,
                                text=' Additional Help Sources ')
@@ -416,11 +414,6 @@
         labelWinHeightTitle = Label(frameWinSize, text='Height')
         entryWinHeight = Entry(
                 frameWinSize, textvariable=self.winHeight, width=3)
-        #paragraphFormatWidth
-        labelParaWidthTitle = Label(
-                frameParaSize, text='Paragraph reformat width (in characters)')
-        entryParaWidth = Entry(
-                frameParaSize, textvariable=self.paraWidth, width=3)
         #frameEncoding
         labelEncodingTitle = Label(
                 frameEncoding, text="Default Source Encoding")
@@ -458,7 +451,6 @@
         frameRun.pack(side=TOP, padx=5, pady=5, fill=X)
         frameSave.pack(side=TOP, padx=5, pady=5, fill=X)
         frameWinSize.pack(side=TOP, padx=5, pady=5, fill=X)
-        frameParaSize.pack(side=TOP, padx=5, pady=5, fill=X)
         frameEncoding.pack(side=TOP, padx=5, pady=5, fill=X)
         frameHelp.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH)
         #frameRun
@@ -475,9 +467,6 @@
         labelWinHeightTitle.pack(side=RIGHT, anchor=E, pady=5)
         entryWinWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         labelWinWidthTitle.pack(side=RIGHT, anchor=E, pady=5)
-        #paragraphFormatWidth
-        labelParaWidthTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
-        entryParaWidth.pack(side=RIGHT, anchor=E, padx=10, pady=5)
         #frameEncoding
         labelEncodingTitle.pack(side=LEFT, anchor=W, padx=5, pady=5)
         radioEncNone.pack(side=RIGHT, anchor=E, pady=5)
@@ -509,7 +498,6 @@
         self.keysAreBuiltin.trace_variable('w', self.VarChanged_keysAreBuiltin)
         self.winWidth.trace_variable('w', self.VarChanged_winWidth)
         self.winHeight.trace_variable('w', self.VarChanged_winHeight)
-        self.paraWidth.trace_variable('w', self.VarChanged_paraWidth)
         self.startupEdit.trace_variable('w', self.VarChanged_startupEdit)
         self.autoSave.trace_variable('w', self.VarChanged_autoSave)
         self.encoding.trace_variable('w', self.VarChanged_encoding)
@@ -594,10 +582,6 @@
         value = self.winHeight.get()
         self.AddChangedItem('main', 'EditorWindow', 'height', value)
 
-    def VarChanged_paraWidth(self, *params):
-        value = self.paraWidth.get()
-        self.AddChangedItem('main', 'FormatParagraph', 'paragraph', value)
-
     def VarChanged_startupEdit(self, *params):
         value = self.startupEdit.get()
         self.AddChangedItem('main', 'General', 'editor-on-startup', value)
@@ -1094,9 +1078,6 @@
                 'main', 'EditorWindow', 'width', type='int'))
         self.winHeight.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'height', type='int'))
-        #initial paragraph reformat size
-        self.paraWidth.set(idleConf.GetOption(
-                'main', 'FormatParagraph', 'paragraph', type='int'))
         # default source encoding
         self.encoding.set(idleConf.GetOption(
                 'main', 'EditorWindow', 'encoding', default='none'))
diff --git a/lib-python/2.7/idlelib/help.txt b/lib-python/2.7/idlelib/help.txt
--- a/lib-python/2.7/idlelib/help.txt
+++ b/lib-python/2.7/idlelib/help.txt
@@ -100,7 +100,7 @@
 			  which is scrolling off the top or the window.
 			  (Not present in Shell window.)
 
-Windows Menu:
+Window Menu:
 
 	Zoom Height -- toggles the window between configured size
 	and maximum height.
diff --git a/lib-python/2.7/idlelib/idle.bat b/lib-python/2.7/idlelib/idle.bat
--- a/lib-python/2.7/idlelib/idle.bat
+++ b/lib-python/2.7/idlelib/idle.bat
@@ -1,4 +1,4 @@
- at echo off
-rem Start IDLE using the appropriate Python interpreter
-set CURRDIR=%~dp0
-start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
+ at echo off
+rem Start IDLE using the appropriate Python interpreter
+set CURRDIR=%~dp0
+start "IDLE" "%CURRDIR%..\..\pythonw.exe" "%CURRDIR%idle.pyw" %1 %2 %3 %4 %5 %6 %7 %8 %9
diff --git a/lib-python/2.7/idlelib/idle_test/test_calltips.py b/lib-python/2.7/idlelib/idle_test/test_calltips.py
--- a/lib-python/2.7/idlelib/idle_test/test_calltips.py
+++ b/lib-python/2.7/idlelib/idle_test/test_calltips.py
@@ -55,7 +55,8 @@
         def gtest(obj, out):
             self.assertEqual(signature(obj), out)
 
-        gtest(List, '()\n' + List.__doc__)
+        if List.__doc__ is not None:
+            gtest(List, '()\n' + List.__doc__)
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
@@ -70,7 +71,8 @@
 
     def test_signature_wrap(self):
         # This is also a test of an old-style class
-        self.assertEqual(signature(textwrap.TextWrapper), '''\
+        if textwrap.TextWrapper.__doc__ is not None:
+            self.assertEqual(signature(textwrap.TextWrapper), '''\
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True)''')
@@ -106,20 +108,23 @@
         def t5(a, b=None, *args, **kwds): 'doc'
         t5.tip = "(a, b=None, *args, **kwargs)"
 
+        doc = '\ndoc' if t1.__doc__ is not None else ''
         for func in (t1, t2, t3, t4, t5, TC):
-            self.assertEqual(signature(func), func.tip + '\ndoc')
+            self.assertEqual(signature(func), func.tip + doc)
 
     def test_methods(self):
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth in (TC.t1, TC.t2, TC.t3, TC.t4, TC.t5, TC.t6, TC.__call__):
-            self.assertEqual(signature(meth), meth.tip + "\ndoc")
-        self.assertEqual(signature(TC.cm), "(a)\ndoc")
-        self.assertEqual(signature(TC.sm), "(b)\ndoc")
+            self.assertEqual(signature(meth), meth.tip + doc)
+        self.assertEqual(signature(TC.cm), "(a)" + doc)
+        self.assertEqual(signature(TC.sm), "(b)" + doc)
 
     def test_bound_methods(self):
         # test that first parameter is correctly removed from argspec
+        doc = '\ndoc' if TC.__doc__ is not None else ''
         for meth, mtip  in ((tc.t1, "()"), (tc.t4, "(*args)"), (tc.t6, "(self)"),
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
-            self.assertEqual(signature(meth), mtip + "\ndoc")
+            self.assertEqual(signature(meth), mtip + doc)
 
     def test_starred_parameter(self):
         # test that starred first parameter is *not* removed from argspec
diff --git a/lib-python/2.7/idlelib/idle_test/test_io.py b/lib-python/2.7/idlelib/idle_test/test_io.py
new file mode 100644
--- /dev/null
+++ b/lib-python/2.7/idlelib/idle_test/test_io.py
@@ -0,0 +1,267 @@
+import unittest
+import io
+from idlelib.PyShell import PseudoInputFile, PseudoOutputFile
+from test import test_support as support
+
+
+class Base(object):
+    def __str__(self):
+        return '%s:str' % type(self).__name__
+    def __unicode__(self):
+        return '%s:unicode' % type(self).__name__
+    def __len__(self):
+        return 3
+    def __iter__(self):
+        return iter('abc')
+    def __getitem__(self, *args):
+        return '%s:item' % type(self).__name__
+    def __getslice__(self, *args):
+        return '%s:slice' % type(self).__name__
+
+class S(Base, str):
+    pass
+
+class U(Base, unicode):
+    pass
+
+class BA(Base, bytearray):
+    pass
+
+class MockShell:
+    def __init__(self):
+        self.reset()
+
+    def write(self, *args):
+        self.written.append(args)
+
+    def readline(self):
+        return self.lines.pop()
+
+    def close(self):
+        pass
+
+    def reset(self):
+        self.written = []
+
+    def push(self, lines):
+        self.lines = list(lines)[::-1]
+
+
+class PseudeOutputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertFalse(f.readable())
+        self.assertTrue(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.read, 0)
+        self.assertRaises(IOError, f.readline, 0)
+
+    def test_write(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.write('test')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        shell.reset()
+        f.write('t\xe8st')
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        shell.reset()
+        f.write(u't\xe8st')
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        shell.reset()
+
+        f.write(S('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(BA('t\xe8st'))
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.write(U(u't\xe8st'))
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.write)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.write, 'test', 'spam')
+        self.assertEqual(shell.written, [])
+
+    def test_writelines(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        f.writelines([])
+        self.assertEqual(shell.written, [])
+        shell.reset()
+        f.writelines(['one\n', 'two'])
+        self.assertEqual(shell.written,
+                         [('one\n', 'stdout'), ('two', 'stdout')])
+        shell.reset()
+        f.writelines(['on\xe8\n', 'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [('on\xe8\n', 'stdout'), ('tw\xf2', 'stdout')])
+        shell.reset()
+        f.writelines([u'on\xe8\n', u'tw\xf2'])
+        self.assertEqual(shell.written,
+                         [(u'on\xe8\n', 'stdout'), (u'tw\xf2', 'stdout')])
+        shell.reset()
+
+        f.writelines([S('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([BA('t\xe8st')])
+        self.assertEqual(shell.written, [('t\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), str)
+        shell.reset()
+        f.writelines([U(u't\xe8st')])
+        self.assertEqual(shell.written, [(u't\xe8st', 'stdout')])
+        self.assertEqual(type(shell.written[0][0]), unicode)
+        shell.reset()
+
+        self.assertRaises(TypeError, f.writelines)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, 123)
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [123])
+        self.assertEqual(shell.written, [])
+        self.assertRaises(TypeError, f.writelines, [], [])
+        self.assertEqual(shell.written, [])
+
+    def test_close(self):
+        shell = MockShell()
+        f = PseudoOutputFile(shell, 'stdout', 'utf-8')
+        self.assertFalse(f.closed)
+        f.write('test')
+        f.close()
+        self.assertTrue(f.closed)
+        self.assertRaises(ValueError, f.write, 'x')
+        self.assertEqual(shell.written, [('test', 'stdout')])
+        f.close()
+        self.assertRaises(TypeError, f.close, 1)
+
+
+class PseudeInputFilesTest(unittest.TestCase):
+    def test_misc(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertIsInstance(f, io.TextIOBase)
+        self.assertEqual(f.encoding, 'utf-8')
+        self.assertIsNone(f.errors)
+        self.assertIsNone(f.newlines)
+        self.assertEqual(f.name, '')
+        self.assertFalse(f.closed)
+        self.assertTrue(f.isatty())
+        self.assertTrue(f.readable())
+        self.assertFalse(f.writable())
+        self.assertFalse(f.seekable())
+
+    def test_unsupported(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        self.assertRaises(IOError, f.fileno)
+        self.assertRaises(IOError, f.tell)
+        self.assertRaises(IOError, f.seek, 0)
+        self.assertRaises(IOError, f.write, 'x')
+        self.assertRaises(IOError, f.writelines, ['x'])
+
+    def test_read(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(-1), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.read(None), 'one\ntwo\n')
+        shell.push(['one\n', 'two\n', 'three\n', ''])
+        self.assertEqual(f.read(2), 'on')
+        self.assertEqual(f.read(3), 'e\nt')
+        self.assertEqual(f.read(10), 'wo\nthree\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.read(0), '')
+        self.assertRaises(TypeError, f.read, 1.5)
+        self.assertRaises(TypeError, f.read, '1')
+        self.assertRaises(TypeError, f.read, 1, 1)
+
+    def test_readline(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', 'three\n', 'four\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(-1), 'two\n')
+        self.assertEqual(f.readline(None), 'three\n')
+        shell.push(['one\ntwo\n'])
+        self.assertEqual(f.readline(), 'one\n')
+        self.assertEqual(f.readline(), 'two\n')
+        shell.push(['one', 'two', 'three'])
+        self.assertEqual(f.readline(), 'one')
+        self.assertEqual(f.readline(), 'two')
+        shell.push(['one\n', 'two\n', 'three\n'])
+        self.assertEqual(f.readline(2), 'on')
+        self.assertEqual(f.readline(1), 'e')
+        self.assertEqual(f.readline(1), '\n')
+        self.assertEqual(f.readline(10), 'two\n')
+
+        shell.push(['one\n', 'two\n'])
+        self.assertEqual(f.readline(0), '')
+        self.assertRaises(TypeError, f.readlines, 1.5)
+        self.assertRaises(TypeError, f.readlines, '1')
+        self.assertRaises(TypeError, f.readlines, 1, 1)
+
+    def test_readlines(self):
+        shell = MockShell()
+        f = PseudoInputFile(shell, 'stdin', 'utf-8')
+        shell.push(['one\n', 'two\n', ''])
+        self.assertEqual(f.readlines(), ['one\n', 'two\n'])
+        shell.push(['one\n', 'two\n', ''])

From noreply at buildbot.pypy.org  Wed Aug 19 20:21:51 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:21:51 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: start 2.6.1 release cycle
Message-ID: <20150819182151.D7AC41C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79070:e40df319ffbe
Date: 2015-08-19 20:53 +0300
http://bitbucket.org/pypy/pypy/changeset/e40df319ffbe/

Log:	start 2.6.1 release cycle

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -168,7 +168,6 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
-  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -215,6 +214,7 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
+  Anton Gulenko
   Gabriel
   Lukas Vacek
   Andrew Dalke
@@ -247,6 +247,7 @@
   Toni Mattis
   Lucas Stadler
   Julian Berman
+  Markus Holtermann
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -32,6 +32,7 @@
   Lukas Diekmann
   Sven Hager
   Anders Lehmann
+  Richard Plangger
   Aurelien Campeas
   Remi Meier
   Niklaus Haldimann
@@ -57,7 +58,6 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
-  Richard Plangger
   Alex Martelli
   Michal Bendowski
   stian
@@ -138,7 +138,6 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
-  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -185,6 +184,7 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
+  Anton Gulenko
   Gabriel
   Lukas Vacek
   Andrew Dalke
@@ -217,6 +217,7 @@
   Toni Mattis
   Lucas Stadler
   Julian Berman
+  Markus Holtermann
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -252,6 +253,7 @@
   shoma hosaka
   Daniel Neuhäuser
   Ben Mather
+  Niclas Olofsson
   halgari
   Boglarka Vezer
   Chris Pressey
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.7.0-alpha0"
+#define PYPY_VERSION "2.6.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (2, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (2, 6, 1, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From noreply at buildbot.pypy.org  Wed Aug 19 20:21:54 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:21:54 +0200 (CEST)
Subject: [pypy-commit] pypy default: move whatsnew
Message-ID: <20150819182154.116C81C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79071:4a4980f065bd
Date: 2015-08-19 20:56 +0300
http://bitbucket.org/pypy/pypy/changeset/4a4980f065bd/

Log:	move whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.6.1.rst
rename from pypy/doc/whatsnew-head.rst
rename to pypy/doc/whatsnew-2.6.1.rst

From noreply at buildbot.pypy.org  Wed Aug 19 20:21:56 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:21:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: start whatsnew-head
Message-ID: <20150819182156.312791C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79072:a7d3e805e03b
Date: 2015-08-19 20:57 +0300
http://bitbucket.org/pypy/pypy/changeset/a7d3e805e03b/

Log:	start whatsnew-head

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-head.rst
@@ -0,0 +1,7 @@
+=======================
+What's new in PyPy 2.6+
+=======================
+
+.. this is a revision shortly after release-2.6.1
+.. startrev: 83ebc73d4fcb
+

From noreply at buildbot.pypy.org  Wed Aug 19 20:21:58 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:21:58 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: merge default into release
Message-ID: <20150819182158.429401C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79073:a462c08f1782
Date: 2015-08-19 20:57 +0300
http://bitbucket.org/pypy/pypy/changeset/a462c08f1782/

Log:	merge default into release

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.6.1.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-2.6.1.rst
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,68 +2,6 @@
 What's new in PyPy 2.6+
 =======================
 
-.. this is a revision shortly after release-2.6.0
-.. startrev: 91904d5c5188
+.. this is a revision shortly after release-2.6.1
+.. startrev: 83ebc73d4fcb
 
-.. branch: use_min_scalar
-Correctly resolve the output dtype of ufunc(array, scalar) calls.
-
-.. branch: stdlib-2.7.10
-
-Update stdlib to version 2.7.10
-
-.. branch: issue2062
-
-.. branch: disable-unroll-for-short-loops
-The JIT no longer performs loop unrolling if the loop compiles to too much code.
-
-.. branch: run-create_cffi_imports
-
-Build cffi import libraries as part of translation by monkey-patching an 
-additional task into translation
-
-.. branch: int-float-list-strategy
-
-Use a compact strategy for Python lists that mix integers and floats,
-at least if the integers fit inside 32 bits.  These lists are now
-stored as an array of floats, like lists that contain only floats; the
-difference is that integers are stored as tagged NaNs.  (This should
-have no visible effect!  After ``lst = [42, 42.5]``, the value of
-``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
-
-.. branch: cffi-callback-onerror
-.. branch: cffi-new-allocator
-
-.. branch: unicode-dtype
-
-Partial implementation of unicode dtype and unicode scalars.
-
-.. branch: dtypes-compatability
-
-Improve compatibility with numpy dtypes; handle offsets to create unions,
-fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
-allow subclassing dtype
-
-.. branch: indexing
-
-Refactor array indexing to support ellipses.
-
-.. branch: numpy-docstrings
-
-Allow the docstrings of built-in numpy objects to be set at run-time.
-
-.. branch: nditer-revisited
-
-Implement nditer 'buffered' flag and fix some edge cases
-
-.. branch: ufunc-reduce
-
-Allow multiple axes in ufunc.reduce()
-
-.. branch: fix-tinylang-goals
-
-Update tinylang goals to match current rpython
-
-.. branch: vmprof-review
-
-Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,12 @@
             i = 0
             count = 0
             i += 5 * WORD # header
-            assert s[i] == '\x04'
-            i += 1 # marker
-            assert s[i] == '\x04'
-            i += 1 # length
-            i += len('pypy')
+            assert s[i    ] == '\x05'    # MARKER_HEADER
+            assert s[i + 1] == '\x00'    # 0
+            assert s[i + 2] == '\x01'    # VERSION_THREAD_ID
+            assert s[i + 3] == chr(4)    # len('pypy')
+            assert s[i + 4: i + 8] == 'pypy'
+            i += 8
             while i < len(s):
                 if s[i] == '\x03':
                     break
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -42,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
+""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,6 +1087,16 @@
     """
     assert type(value) is cls
 
+def ll_record_exact_class(ll_value, ll_cls):
+    from rpython.rlib.debug import ll_assert
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.rclass import ll_type
+    ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument")
+    ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments")
+    llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls)
+
+
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_class
 
@@ -1100,12 +1110,10 @@
         from rpython.rtyper import rclass
 
         classrepr = rclass.get_type_repr(hop.rtyper)
-
-        hop.exception_cannot_occur()
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
         v_cls = hop.inputarg(classrepr, arg=1)
-        return hop.genop('jit_record_exact_class', [v_inst, v_cls],
-                         resulttype=lltype.Void)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
     pass
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -616,26 +616,33 @@
                 while rbase.classdef is not None:
                     immutable_fields.update(rbase.immutable_field_set)
                     rbase = rbase.rbase
-                self._parse_field_list(immutable_fields, accessor)
+                self._parse_field_list(immutable_fields, accessor, hints)
 
-    def _parse_field_list(self, fields, accessor):
+    def _parse_field_list(self, fields, accessor, hints):
         ranking = {}
         for name in fields:
+            quasi = False
             if name.endswith('?[*]'):   # a quasi-immutable field pointing to
                 name = name[:-4]        # an immutable array
                 rank = IR_QUASIIMMUTABLE_ARRAY
+                quasi = True
             elif name.endswith('[*]'):    # for virtualizables' lists
                 name = name[:-3]
                 rank = IR_IMMUTABLE_ARRAY
             elif name.endswith('?'):    # a quasi-immutable field
                 name = name[:-1]
                 rank = IR_QUASIIMMUTABLE
+                quasi = True
             else:                       # a regular immutable/green field
                 rank = IR_IMMUTABLE
             try:
                 mangled_name, r = self._get_field(name)
             except KeyError:
                 continue
+            if quasi and hints.get("immutable"):
+                raise TyperError(
+                    "can't have _immutable_ = True and a quasi-immutable field "
+                    "%s in class %s" % (name, self.classdef))
             ranking[mangled_name] = rank
         accessor.initialize(self.object_type, ranking)
         return ranking
diff --git a/rpython/rtyper/rvirtualizable.py b/rpython/rtyper/rvirtualizable.py
--- a/rpython/rtyper/rvirtualizable.py
+++ b/rpython/rtyper/rvirtualizable.py
@@ -38,8 +38,8 @@
             else:
                 self._super()._setup_repr(hints = hints)
             c_vfields = self.classdef.classdesc.classdict['_virtualizable_']
-            self.my_redirected_fields = self._parse_field_list(c_vfields.value,
-                                                               self.accessor)
+            self.my_redirected_fields = self._parse_field_list(
+                c_vfields.value, self.accessor, hints)
         else:
             self._super()._setup_repr()
             # ootype needs my_redirected_fields even for subclass. lltype does
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -943,6 +943,19 @@
                 found.append(op.args[1].value)
         assert found == ['mutate_a', 'mutate_a', 'mutate_b']
 
+    def test_quasi_immutable_clashes_with_immutable(self):
+        from rpython.jit.metainterp.typesystem import deref
+        class A(object):
+            _immutable_ = True
+            _immutable_fields_ = ['a?']
+        def f():
+            a = A()
+            a.x = 42
+            a.a = 142
+            return A()
+        with py.test.raises(TyperError):
+            self.gengraph(f, [])
+
     def test_quasi_immutable_array(self):
         from rpython.jit.metainterp.typesystem import deref
         class A(object):

From noreply at buildbot.pypy.org  Wed Aug 19 20:22:00 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:22:00 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: typo, start release notice
Message-ID: <20150819182200.49E101C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79074:bb49ec3f98b6
Date: 2015-08-19 21:19 +0300
http://bitbucket.org/pypy/pypy/changeset/bb49ec3f98b6/

Log:	typo, start release notice

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.6.1.rst
@@ -0,0 +1,114 @@
+==========
+PyPy 2.6.1 
+==========
+
+We're pleased to announce PyPy 2.6.1, an update to PyPy 2.6.0 released June 1.
+We have updated stdlib to 2.7.10, `cffi`_ to version 1.2, extended support for
+the new vmprof_ statistical profiler for multiple threads, partially
+implemented unicode dtypes in numpy, added support for the axis keyword to
+many numpy methods, and in general improved numpy compatability.
+
+You can download the PyPy 2.6.1 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and for those who donate to our three sub-projects, as well as our
+volunteers and contributors.  
+
+We've shown quite a bit of progress, but we're slowly running out of funds.
+Please consider donating more, or even better convince your employer to donate,
+so we can finish those projects! The three sub-projects are:
+
+* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
+  we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version
+
+* `STM`_ (software transactional memory): We have released a first working version,
+  and continue to try out new promising paths of achieving a fast multithreaded Python
+
+* `NumPy`_ which requires installation of our fork of upstream numpy,
+  available `on bitbucket`_
+
+.. _`cffi`: https://cffi.readthedocs.org
+.. _`Py3k`: http://pypy.org/py3donate.html
+.. _`STM`: http://pypy.org/tmdonate2.html
+.. _`NumPy`: http://pypy.org/numpydonate.html
+.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy
+
+We would also like to encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making
+RPython's JIT even better. 
+
+.. _`PyPy`: http://doc.pypy.org 
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems
+(Linux 32/64, Mac OS X 64, Windows, OpenBSD_, freebsd_),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux.
+
+While we support 32 bit python on Windows, work on the native Windows 64
+bit python is still stalling, we would welcome a volunteer 
+to `handle that`_. We also welcome developers with other operating systems or
+`dynamic languages`_ to see what RPython can do for them.
+
+.. _`pypy and cpython 2.7.x`: http://speed.pypy.org
+.. _OpenBSD: http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/lang/pypy
+.. _freebsd: https://svnweb.freebsd.org/ports/head/lang/pypy/
+.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
+.. _`dynamic languages`: http://pypyjs.org
+
+Highlights 
+===========
+
+* Bug Fixes
+
+  * 
+
+  * 
+
+  * Issues reported with our previous release were resolved_ after reports from users on
+    our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+    #pypy.
+
+* New features:
+
+  * 
+
+* Numpy:
+
+  * 
+
+  * 
+
+* Performance improvements:
+
+  * 
+
+  * 
+
+  * 
+
+  * 
+
+.. _`vmprof`: https://vmprof.readthedocs.org
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.0.html
+
+Please try it out and let us know what you think. We welcome
+success stories, `experiments`_,  or `benchmarks`_, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+
+.. _`experiments`: https://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html
+.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -1,6 +1,6 @@
-=======================
-What's new in PyPy 2.6+
-=======================
+========================
+What's new in PyPy 2.6.1
+========================
 
 .. this is a revision shortly after release-2.6.0
 .. startrev: 91904d5c5188

From noreply at buildbot.pypy.org  Wed Aug 19 20:22:02 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:22:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: typo
Message-ID: <20150819182202.5D7161C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79075:59d9c4addd35
Date: 2015-08-19 21:20 +0300
http://bitbucket.org/pypy/pypy/changeset/59d9c4addd35/

Log:	typo

diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -1,6 +1,6 @@
-=======================
-What's new in PyPy 2.6+
-=======================
+========================
+What's new in PyPy 2.6.1
+========================
 
 .. this is a revision shortly after release-2.6.0
 .. startrev: 91904d5c5188

From noreply at buildbot.pypy.org  Wed Aug 19 20:28:22 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 19 Aug 2015 20:28:22 +0200 (CEST)
Subject: [pypy-commit] pypy default: create link in docs
Message-ID: <20150819182822.E6D631C1186@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79076:11afba3fa8d5
Date: 2015-08-19 21:27 +0300
http://bitbucket.org/pypy/pypy/changeset/11afba3fa8d5/

Log:	create link in docs

diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.6.1.rst
    whatsnew-2.6.0.rst
    whatsnew-2.5.1.rst
    whatsnew-2.5.0.rst

From noreply at buildbot.pypy.org  Wed Aug 19 21:00:12 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 21:00:12 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start fighting with
	test_virtualstate
Message-ID: <20150819190012.F23C41C0F47@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79077:8b79d27afa90
Date: 2015-08-19 21:00 +0200
http://bitbucket.org/pypy/pypy/changeset/8b79d27afa90/

Log:	start fighting with test_virtualstate

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -38,6 +38,9 @@
     def get_known_class(self, cpu):
         return None
 
+    def getlenbound(self):
+        return None
+
     def getnullness(self):
         if self.is_null():
             return INFO_NULL
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2733,7 +2733,7 @@
             '''
             p1.nextdescr = p2
             where p2 is a node_vtable, valuedescr=i2
-            ''', rop.GUARD_TRUE, values=[InputArgInt(0),
+            ''', rop.GUARD_TRUE, values=[InputArgInt(18),
                                          InputArgRef(self.nodeaddr)])
 
     def test_expand_fail_lazy_setfield_2(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -8,6 +8,7 @@
 from rpython.jit.metainterp.resoperation import InputArgInt, InputArgRef,\
      InputArgFloat
 from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper import rclass
 from rpython.jit.metainterp.optimizeopt.test.test_util import LLtypeMixin, BaseTest, \
                                                            equaloplists
 from rpython.jit.metainterp.optimizeopt.intutils import IntBound,\
@@ -25,10 +26,13 @@
 
 class BaseTestGenerateGuards(BaseTest):
 
-    def _box_or_value(self, box_or_value=None):
-        if box_or_value is None:
+    def _box_or_value(self, boxinfo=None):
+        if boxinfo is None:
             return None, None
-        elif isinstance(box_or_value, OptValue):
+        else:
+            xxx
+        # el
+        if isinstance(box_or_value, OptValue):
             value = box_or_value
             box = value.box
         else:
@@ -57,19 +61,21 @@
         assert equaloplists(guards, loop.operations, False,
                             boxmap)
 
-    def check_no_guards(self, info1, info2, box_or_value=None, state=None):
-        value, _ = self._box_or_value(box_or_value)
+    def check_no_guards(self, info1, info2, boxinfo=None, state=None):
+        assert boxinfo is None
+        boxinfo, _ = self._box_or_value(boxinfo)
         if info1.position == -1:
             info1.position = 0
         if info2.position == -1:
             info2.position = 0
         if state is None:
             state = GenerateGuardState(self.cpu)
-        info1.generate_guards(info2, value, state)
+        info1.generate_guards(info2, boxinfo, None, state)
         assert not state.extra_guards
         return state
 
     def check_invalid(self, info1, info2, box_or_value=None, state=None):
+        assert box_or_value is None
         value, _ = self._box_or_value(box_or_value)
         if info1.position == -1:
             info1.position = 0
@@ -78,7 +84,7 @@
         if state is None:
             state = GenerateGuardState(self.cpu)
         with py.test.raises(VirtualStatesCantMatch):
-            info1.generate_guards(info2, value, state)
+            info1.generate_guards(info2, None, None, state)
 
     def test_make_inputargs(self):
         optimizer = FakeOptimizer()
@@ -150,22 +156,23 @@
                 if i != j:
                     assert not isgeneral('r', inorder[j], 'r', inorder[i])
 
-        i1 = IntLowerBound(10)
-        i2 = IntUnbounded()
+        i1 = IntUnbounded()
+        i2 = IntLowerBound(10)
         assert isgeneral('i', i1, 'i', i2)
         assert not isgeneral('i', i2, 'i', i1)
 
-        assert isgeneral(OptValue(ConstInt(7)), OptValue(ConstInt(7)))
-        S = lltype.GcStruct('S')
+        assert isgeneral('i', ConstIntBound(7), 'i', ConstIntBound(7))
+        S = lltype.GcStruct('S', ('parent', rclass.OBJECT))
         foo = lltype.malloc(S)
+        foo_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
+        foo.parent.typeptr = foo_vtable
         fooref = lltype.cast_opaque_ptr(llmemory.GCREF, foo)
-        assert isgeneral(OptValue(ConstPtr(fooref)),
-                         OptValue(ConstPtr(fooref)))
+        assert isgeneral('r', info.ConstPtrInfo(ConstPtr(fooref)),
+                         'r', info.ConstPtrInfo(ConstPtr(fooref)))
 
-        value1 = PtrOptValue(BoxPtr())
-        value1.make_nonnull(None)
-        value2 = PtrOptValue(ConstPtr(self.nullptr))
-        assert not isgeneral(value1, value2)
+        value1 = info.NonNullPtrInfo()
+        value2 = info.ConstPtrInfo(ConstPtr(self.nullptr))
+        assert not isgeneral('r', value1, 'r', value2)
 
     def test_field_matching_generalization(self):
         const1 = NotVirtualStateInfo(OptValue(ConstInt(1)))
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,6 +1,7 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp.history import (ConstInt, Const,
         ConstPtr, ConstFloat)
+from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
@@ -36,17 +37,17 @@
 class AbstractVirtualStateInfo(object):
     position = -1
 
-    def generate_guards(self, other, value, state):
+    def generate_guards(self, other, op, opinfo, state):
         """ generate guards (output in the list extra_guards) that make runtime
         values of the shape other match the shape of self. if that's not
         possible, VirtualStatesCantMatch is thrown and bad gets keys set which
         parts of the state are the problem.
 
-        the function can peek into value (and particularly also the boxes in
-        the value) as a guiding heuristic whether making such guards makes
-        sense. if None is passed in for value, no guard is ever generated, and
+        the function can peek into opinfo (and particularly also the op)
+        as a guiding heuristic whether making such guards makes
+        sense. if None is passed in for op, no guard is ever generated, and
         this function degenerates to a generalization check."""
-        assert value is None or isinstance(value, OptValue)
+        assert opinfo is None or isinstance(opinfo, info.AbstractInfo)
         assert self.position != -1
         if self.position in state.renum:
             if state.renum[self.position] != other.position:
@@ -60,7 +61,7 @@
         else:
             state.renum[self.position] = other.position
             try:
-                self._generate_guards(other, value, state)
+                self._generate_guards(other, op, opinfo, state)
             except VirtualStatesCantMatch, e:
                 state.bad[self] = state.bad[other] = None
                 if e.state is None:
@@ -105,14 +106,15 @@
     def __init__(self, fielddescrs):
         self.fielddescrs = fielddescrs
 
-    def _generate_guards(self, other, value, state):
+    def _generate_guards(self, other, box, opinfo, state):
         if not self._generalization_of_structpart(other):
             raise VirtualStatesCantMatch("different kinds of structs")
 
         assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
-        if value is not None:
+        if box is not None:
+            yyy
             assert isinstance(value, virtualize.AbstractVirtualStructValue)
             assert value.is_virtual()
 
@@ -120,6 +122,7 @@
             raise VirtualStatesCantMatch("field descrs don't match")
 
         for i in range(len(self.fielddescrs)):
+            yyy
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 raise VirtualStatesCantMatch("field descrs don't match")
             if value is not None:
@@ -186,7 +189,7 @@
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
 
-    def _generate_guards(self, other, value, state):
+    def _generate_guards(self, other, box, opinfo, state):
         if not isinstance(other, VArrayStateInfo):
             raise VirtualStatesCantMatch("other is not an array")
         if self.arraydescr is not other.arraydescr:
@@ -195,6 +198,7 @@
             raise VirtualStatesCantMatch("other has a different length")
         v = None
         for i in range(len(self.fieldstate)):
+            xxxx
             if value is not None:
                 assert isinstance(value, virtualize.VArrayValue)
                 v = value._items[i]
@@ -290,48 +294,33 @@
     constbox = None
     known_class = None
     
-    def __init__(self, cpu, type, info):
+    def __init__(self, cpu, type, info, is_opaque=False):
+        self.is_opaque = is_opaque
         if info and info.is_constant():
+            self.level = LEVEL_CONSTANT
             self.constbox = info.getconst()
-            self.level = LEVEL_CONSTANT
-        elif type == 'r' and info:
-            if info.get_known_class(cpu):
+            if type == 'r':
                 self.known_class = info.get_known_class(cpu)
-                self.level = LEVEL_KNOWNCLASS
-            elif info.is_nonnull():
-                self.level = LEVEL_NONNULL
-        return
+        elif type == 'r':
+            if info:
+                self.known_class = info.get_known_class(cpu)
+                if self.known_class:
+                    self.level = LEVEL_KNOWNCLASS
+                elif info.is_nonnull():
+                    self.level = LEVEL_NONNULL
+                self.lenbound = info.getlenbound()
+        elif type == 'i':
+            self.intbound = info
 
     def is_const(self):
         return self.constbox is not None
-        yyy
-        self.level = LEVEL_UNKNOWN
-        if ptrinfo is not None:
-            self.known_class = ptrinfo.get_known_class(cpu)
-        return
-        xxx
-        self.is_opaque = is_opaque
-        self.known_class = value.get_known_class()
-        self.level = value.getlevel()
-        if value.getintbound() is None:
-            self.intbound = IntUnbounded()
-        else:
-            self.intbound = value.getintbound().clone()
-        if value.is_constant():
-            self.constbox = value.box
-        else:
-            self.constbox = None
-        self.position_in_notvirtuals = -1
-        self.lenbound = value.getlenbound()
 
     def is_virtual(self):
         return False
 
-    def _generate_guards(self, other, value, state):
-        if value is None or self.is_opaque:
+    def _generate_guards(self, other, box, opinfo, state):
+        if self.is_opaque:
             box = None # generating guards for opaque pointers isn't safe
-        else:
-            box = value.box
         # XXX This will always retrace instead of forcing anything which
         # might be what we want sometimes?
         if not isinstance(other, NotVirtualStateInfo):
@@ -417,13 +406,12 @@
                 raise VirtualStatesCantMatch("other not constant")
         assert 0, "unreachable"
 
-    def _generate_guards_intbounds(self, other, box, extra_guards):
+    def _generate_guards_intbounds(self, other, boxinfo, extra_guards):
         if self.intbound is None:
             return
         if self.intbound.contains_bound(other.intbound):
             return
-        xxx
-        if (box is not None and isinstance(box, BoxInt) and
+        if (boxinfo is not None and isinstance(box, BoxInt) and
                 self.intbound.contains(box.getint())):
             # this may generate a few more guards than needed, but they are
             # optimized away when emitting them
@@ -503,7 +491,7 @@
         assert len(self.state) == len(other.state)
         try:
             for i in range(len(self.state)):
-                self.state[i].generate_guards(other.state[i], None, state)
+                self.state[i].generate_guards(other.state[i], None, None, state)
         except VirtualStatesCantMatch:
             return False
         return True

From noreply at buildbot.pypy.org  Wed Aug 19 21:31:33 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 19 Aug 2015 21:31:33 +0200 (CEST)
Subject: [pypy-commit] pypy default: hack for MAC
Message-ID: <20150819193133.DBCA11C1186@cobra.cs.uni-duesseldorf.de>

Author: fijal
Branch: 
Changeset: r79078:a3278fa0fd93
Date: 2015-08-19 21:30 +0200
http://bitbucket.org/pypy/pypy/changeset/a3278fa0fd93/

Log:	hack for MAC

diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -53,7 +53,9 @@
 // If #define _GNU_SOURCE causes problems, this might work instead.
 // It will cause problems for FreeBSD though!, because it turns off
 // the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
+#ifdef __APPLE__
+#define _XOPEN_SOURCE 500
+#endif
 
 #include          // for memcmp
 #if defined(HAVE_SYS_UCONTEXT_H)
@@ -179,7 +181,11 @@
 // configure.ac (or set it manually in your config.h).
 #else
 void* GetPC(ucontext_t *signal_ucontext) {
+#ifdef __APPLE__
+  return (void*)(signal_ucontext->uc_mcontext->__ss.__rip);
+#else
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
+#endif
 }
 
 #endif
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -27,6 +27,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include "vmprof_getpc.h"
@@ -457,7 +458,7 @@
     if (!srcf)
         return -1;
 
-    while ((size = fread(buf, 1, sizeof buf, src))) {
+    while ((size = fread(buf, 1, sizeof buf, srcf))) {
         if (_write_all(buf, size) < 0) {
             pclose(srcf);
             return -1;
diff --git a/rpython/rlib/rvmprof/src/vmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
--- a/rpython/rlib/rvmprof/src/vmprof_mt.h
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -66,7 +66,7 @@
     unprepare_concurrent_bufs();
     profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
                                PROT_READ | PROT_WRITE,
-                               MAP_PRIVATE | MAP_ANONYMOUS,
+                               MAP_PRIVATE | MAP_ANON,
                                -1, 0);
     if (profbuf_all_buffers == MAP_FAILED) {
         profbuf_all_buffers = NULL;

From noreply at buildbot.pypy.org  Thu Aug 20 08:13:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Thu, 20 Aug 2015 08:13:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: try a hopefully more robust approach
Message-ID: <20150820061311.6B3C21C0933@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79079:d5e366cc2f51
Date: 2015-08-20 09:13 +0300
http://bitbucket.org/pypy/pypy/changeset/d5e366cc2f51/

Log:	try a hopefully more robust approach

diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,5 +1,4 @@
 import time, os, sys
-sys.path += ['.'] # for subprocess in test_interpreted
 import py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
@@ -59,7 +58,8 @@
 def test_interpreted():
     # takes forever if the Python process is already big...
     import subprocess
-    subprocess.check_call([sys.executable, __file__])
+    env = os.environ.copy()
+    subprocess.check_call([sys.executable, __file__],env=env)
 
 def test_compiled():
     fn = compile(main, [], gcpolicy="minimark")

From noreply at buildbot.pypy.org  Thu Aug 20 09:34:39 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 20 Aug 2015 09:34:39 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: wrong target token has been added
 to the label jump in case a second label is introduced
Message-ID: <20150820073439.A53251C0F47@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79080:963383c4adcd
Date: 2015-08-20 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/963383c4adcd/

Log:	wrong target token has been added to the label jump in case a second
	label is introduced test for it and some documentation

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -161,10 +161,9 @@
     assert part.operations[-1].getopnum() != rop.LABEL
 
     if loop.versions is not None:
-        # several different loop version have been generated
+        # every different loop version must update their target tokens
         for version in loop.versions:
-            token = version.update_token(jitcell_token)
-            all_target_tokens.append(token)
+            version.update_token(jitcell_token, all_target_tokens)
 
     if not loop.quasi_immutable_deps:
         loop.quasi_immutable_deps = None
@@ -186,6 +185,9 @@
     return all_target_tokens[0]
 
 def generate_pending_loop_versions(loop, jitdriver_sd, metainterp, jitcell_token):
+    """ if a loop version is created for a guard instruction (e.g. they are known
+        to fail frequently, then a version can be created that is immediatly compiled.
+    """
     metainterp_sd = metainterp.staticdata
     cpu = metainterp_sd.cpu
     if loop.versions is not None:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -801,20 +801,33 @@
         op.setfailargs(self.inputargs)
         op.rd_snapshot = None
 
-    def update_token(self, jitcell_token):
+    def update_token(self, jitcell_token, all_target_tokens):
         # this is only invoked for versioned loops!
-        label = self.operations[self.label_pos]
+        label_index = index_of_first(rop.LABEL, self.operations, 0)
+        label = self.operations[label_index]
         jump = self.operations[-1]
         #
-        assert label.getopnum() == rop.LABEL
         assert jump.getopnum() == rop.JUMP
         #
         token = TargetToken(jitcell_token)
         token.original_jitcell_token = jitcell_token
+        all_target_tokens.append(token)
+        if label.getdescr() is not jump.getdescr():
+            label_index = index_of_first(rop.LABEL, self.operations, 1)
+            if label_index > 0:
+                second_label = self.operations[label_index]
+                # set the inner loop
+                second_label.setdescr(token)
+                jump.setdescr(token)
+                # set the first label
+                token = TargetToken(jitcell_token)
+                token.original_jitcell_token = jitcell_token
+                all_target_tokens.append(token)
+                label.setdescr(token)
+                return
         label.setdescr(token)
         jump.setdescr(token)
 
-        return token
 
 class TreeLoop(object):
     inputargs = None
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -876,8 +876,8 @@
             descr = label.getdescr()
             assert isinstance(descr, TargetToken)
             token = TargetToken(descr.targeting_jitcell_token)
-            oplist[0] = label.copy_and_change(label.getopnum(), label_args, None, token)
-            oplist[-1] = jump.copy_and_change(jump.getopnum(), jump_args, None, token)
+            oplist[0] = label.copy_and_change(label.getopnum(), args=label_args, descr=token)
+            oplist[-1] = jump.copy_and_change(jump.getopnum(), args=jump_args, descr=token)
             #
             return [ResOperation(rop.LABEL, orig_label_args, None, descr)] + \
                    self.invariant_oplist + oplist
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1389,113 +1389,30 @@
 
     def test_1(self):
         trace = """
-        [p0, p1, p5, p6, p7, p9, p11, p12]
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #34 FOR_ITER')
-        guard_early_exit(descr=) [p1, p0, p5, p6, p7, p9]
-        p13 = getfield_gc(p9, descr=)
-        guard_nonnull(p13, descr=) [p1, p0, p9, p13, p5, p6, p7]
-        i14 = getfield_gc(p9, descr=)
-        p15 = getfield_gc(p13, descr=)
-        guard_class(p15, 140737326900656, descr=) [p1, p0, p9, i14, p15, p13, p5, p6, p7]
-        p17 = getfield_gc(p13, descr=)
-        i18 = getfield_gc_pure(p17, descr=)
-        i20 = int_lt(i14, 0)
-        guard_false(i20, descr=) [p1, p0, p9, i14, i18, p5, p6, p7]
-        i21 = int_ge(i14, i18)
-        guard_false(i21, descr=) [p1, p0, p9, i14, p5, p6, p7]
-        i23 = int_add(i14, 1)
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #37 STORE_NAME')
-        p24 = getfield_gc(p5, descr=)
-        setfield_gc(p9, i23, descr=)
-        guard_value(p24, 25, descr=) [p1, p0, p24, p5, p6, p7, p9, i14]
-        p26 = getfield_gc(p24, descr=)
-        guard_value(p26, 27, descr=) [p1, p0, p26, p24, p5, p6, p7, p9, i14]
-        guard_not_invalidated(descr=) [p1, p0, p24, p5, p6, p7, p9, i14]
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #40 LOAD_NAME')
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #43 CALL_FUNCTION')
-        p28 = force_token()
-        enter_portal_frame(15, 8070450532247933488)
-        debug_merge_point(1, 1, ' #0 LOAD_CONST')
-        debug_merge_point(1, 1, ' #3 STORE_FAST')
-        debug_merge_point(1, 1, ' #6 SETUP_LOOP')
-        debug_merge_point(1, 1, ' #9 LOAD_FAST')
-        debug_merge_point(1, 1, ' #12 LOAD_CONST')
-        debug_merge_point(1, 1, ' #15 COMPARE_OP')
-        debug_merge_point(1, 1, ' #18 POP_JUMP_IF_FALSE')
-        debug_merge_point(1, 1, ' #21 LOAD_GLOBAL')
-        debug_merge_point(1, 1, ' #24 LOAD_FAST')
-        debug_merge_point(1, 1, ' #27 BINARY_SUBSCR')
-        p32 = getfield_gc(31, descr=)
-        setfield_gc(33, i14, descr=)
-        guard_class(p32, 140737326657856, descr=) [p1, p0, p11, p32, p5, p6, p7, p9, p28, p12, None]
-        p36 = getfield_gc(31, descr=)
-        i37 = getfield_gc(p36, descr=)
-        i39 = uint_ge(0, i37)
-        guard_false(i39, descr=) [p1, p0, p11, i37, p36, p5, p6, p7, p9, p28, p12, None]
-        p40 = getfield_gc(p36, descr=)
-        i41 = getarrayitem_gc(p40, 0, descr=intarraydescr)
-        debug_merge_point(1, 1, ' #28 LOAD_CONST')
-        debug_merge_point(1, 1, ' #31 BINARY_ADD')
-        i43 = int_add_ovf(i41, 1)
-        guard_no_overflow(descr=) [p1, p0, p11, i43, p5, p6, p7, p9, i41, p28, p12, None]
-        debug_merge_point(1, 1, ' #32 LOAD_GLOBAL')
-        debug_merge_point(1, 1, ' #35 LOAD_FAST')
-        debug_merge_point(1, 1, ' #38 STORE_SUBSCR')
-        debug_merge_point(1, 1, ' #39 LOAD_FAST')
-        debug_merge_point(1, 1, ' #42 LOAD_CONST')
-        debug_merge_point(1, 1, ' #45 INPLACE_ADD')
-        debug_merge_point(1, 1, ' #46 STORE_FAST')
-        debug_merge_point(1, 1, ' #49 JUMP_ABSOLUTE')
-        i45 = getfield_raw(140737351875840, descr=)
-        setarrayitem_gc(p40, 0, i43, descr=intarraydescr)
-        i48 = int_lt(i45, 0)
-        guard_false(i48, descr=) [p1, p0, p11, p5, p6, p7, p9, None, p28, p12, None]
-        debug_merge_point(1, 1, ' #9 LOAD_FAST')
-        p49 = force_token()
-        p51 = new_with_vtable(140737326477112)
-        p53 = new_array_clear(4, descr=)
-        p55 = new_with_vtable(100000)
-        p57 = new_with_vtable(140737326516800)
-        setfield_gc(p57, p28, descr=)
-        setfield_gc(p57, 0, descr=)
-        setfield_gc(p11, p57, descr=)
-        setfield_gc(p0, p49, descr=)
-        setfield_gc(p51, 0, descr=)
-        setfield_gc(p51, 0, descr=)
-        setfield_gc(p51, 9, descr=)
-        setfield_gc(p51, 1, descr=)
-        setfield_gc(p51, 25, descr=)
-        setfield_gc(p51, 63, descr=)
-        setfield_gc(p55, 1, descr=)
-        setarrayitem_gc(p53, 0, p55, descr=)
-        setfield_gc(p51, p53, descr=)
-        setfield_gc(p51, p12, descr=)
-        setfield_gc(p51, 66, descr=)
-        p67 = call_assembler(p51, p11, descr=)
-        guard_not_forced(descr=) [p1, p0, p11, p51, p67, p57, p5, p6, p7, p9]
-        keepalive(p51)
-        guard_no_exception(descr=) [p1, p0, p11, p51, p67, p57, p5, p6, p7, p9]
-        leave_portal_frame(15)
-        p69 = getfield_gc(p11, descr=)
-        p70 = getfield_gc(p51, descr=)
-        i71 = getfield_gc(p51, descr=)
-        setfield_gc(p11, p70, descr=)
-        guard_false(i71, descr=) [p1, p0, p67, p69, p51, p57, p5, p6, p7, p9]
-        i72 = getfield_gc(p5, descr=)
-        setfield_gc(p57, 0, descr=)
-        guard_value(i72, 0, descr=) [i72, p1, p0, p5, p6, p7, p9, p67]
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #46 POP_TOP')
-        p75 = getfield_gc(p5, descr=)
-        guard_isnull(p75, descr=) [p1, p0, p75, p5, p6, p7, p9, p67]
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #47 JUMP_ABSOLUTE')
-        guard_not_invalidated(descr=) [p1, p0, p5, p6, p7, p9]
-        i77 = getfield_raw(140737351875840, descr=)
-        i79 = int_lt(i77, 0)
-        guard_false(i79, descr=) [p1, p0, p5, p6, p7, p9]
-        debug_merge_point(0, 0, '. file '/home/rich/fijal.py'. line 2> #34 FOR_ITER')
-        jump(p0, p1, p5, p6, p7, p9, p11, p70)
+        [p0, p1, p6, p7, i13, p14, p15]
+        guard_early_exit(descr=) [p1, p0, p6, p7, i13]
+        guard_not_invalidated(descr=) [p1, p0, p6, p7, i13]
+        i17 = int_lt(i13, 10000)
+        guard_true(i17, descr=) [p1, p0, p6, p7, i13]
+        i18 = getfield_gc(p14, descr=)
+        i19 = uint_ge(i13, i18)
+        guard_false(i19, descr=) [p1, p0, i18, i13, p14, p6, p7, None]
+        p21 = getfield_gc(p14, descr=)
+        f22 = getarrayitem_gc(p21, i13, descr=)
+        i23 = getfield_gc(p15, descr=)
+        i24 = uint_ge(i13, i23)
+        guard_false(i24, descr=) [p1, p0, i23, i13, p15, p6, p7, f22, None]
+        p25 = getfield_gc(p15, descr=)
+        f26 = getarrayitem_gc(p25, i13, descr=floatarraydescr)
+        f27 = float_add(f22, f26)
+        setarrayitem_gc(p21, i13, f27, descr=floatarraydescr)
+        i29 = int_add(i13, 1)
+        #i31 = getfield_raw(140229696280448, descr=)
+        i33 = int_lt(0, 1)
+        guard_false(i33, descr=) [p1, p0, p6, p7, i29, None, None]
+        jump(p0, p1, p6, p7, i29, p14, p15)
         """        
-        #opt = self.schedule(self.parse_loop(trace))
+        opt = self.schedule(self.parse_loop(trace))
         #self.debug_print_operations(opt.loop)
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -19,12 +19,12 @@
     def setup_method(self, method):
         print "RUNNING", method.__name__
 
-    def meta_interp(self, f, args, policy=None):
+    def meta_interp(self, f, args, policy=None, vec=True, vec_all=False):
         return ll_meta_interp(f, args, enable_opts=self.enable_opts,
                               policy=policy,
                               CPUClass=self.CPUClass,
                               type_system=self.type_system,
-                              vec=True)
+                              vec=vec, vec_all=vec_all)
 
     @py.test.mark.parametrize('i',[3,4,5,6,7,8,9,50])
     def test_vectorize_simple_load_arith_store_int_add_index(self,i):
@@ -154,7 +154,6 @@
 
     def test_sum(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
-        myjitdriver2 = JitDriver(greens = [], reds = 'auto', vectorize=True)
         T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
         def f(d):
             va = lltype.malloc(T, d, flavor='raw', zero=True)
@@ -171,6 +170,56 @@
         res = self.meta_interp(f, [60])
         assert res == f(60) == sum(range(60))
 
+    def test_constant_expand(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
+        def f(d):
+            va = lltype.malloc(T, d, flavor='raw', zero=True)
+            i = 0
+            while i < d:
+                myjitdriver.jit_merge_point()
+                va[i] = va[i] + 34.5
+                i += 1
+            val = va[0]
+            lltype.free(va, flavor='raw')
+            return val
+        res = self.meta_interp(f, [60])
+        assert res == f(60) == 34.5
+
+    def test_constant_expand_vec_all(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto')
+        T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
+        def f(d):
+            va = lltype.malloc(T, d, flavor='raw', zero=True)
+            i = 0
+            while i < d:
+                myjitdriver.jit_merge_point()
+                if not (i < 60):
+                    raise IndexError
+                va[i] = va[i] + 34.5
+                i += 1
+            val = va[0]
+            lltype.free(va, flavor='raw')
+            return val
+        res = self.meta_interp(f, [60], vec_all=True)
+        assert res == f(60) == 34.5
+
+    def test_variable_expand(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+        T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
+        def f(d,variable):
+            va = lltype.malloc(T, d, flavor='raw', zero=True)
+            i = 0
+            while i < d:
+                myjitdriver.jit_merge_point()
+                va[i] = va[i] + variable
+                i += 1
+            val = va[0]
+            lltype.free(va, flavor='raw')
+            return val
+        res = self.meta_interp(f, [60,58.4547])
+        assert res == f(60,58.4547) == 58.4547
+
     @py.test.mark.parametrize('i',[15])
     def test_array_bounds_check_elimination(self,i):
         myjitdriver = JitDriver(greens = [],
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -72,7 +72,7 @@
                     loop_longevity=0, retrace_limit=5, function_threshold=4,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
                     max_unroll_recursion=7, vec=0, vec_all=0, vec_cost=0,
-                    vec_length=50, vec_ratio=6, **kwds):
+                    vec_length=60, vec_ratio=2, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -554,10 +554,10 @@
                    'optimizations to enable, or all = %s' % ENABLE_ALL_OPTS,
     'max_unroll_recursion': 'how many levels deep to unroll a recursive function',
     'vec': 'turn on the vectorization optimization (vecopt). requires sse4.1',
-    'vec_all': 'all = 1: try to vectorize trace loops that occur outside of the numpy library.',
-    'vec_cost': 'cost = 0: threshold for which traces to bail. 0 means the costs.',
-    'vec_length': 'length = 50:  the amount of instructions allowed in "all" traces.',
-    'vec_ratio': 'ratio = 6: an integer (0-10 => X / 10) statements that have vector equivalents '
+    'vec_all': 'try to vectorize trace loops that occur outside of the numpy library.',
+    'vec_cost': 'threshold for which traces to bail. 0 means the costs.',
+    'vec_length': 'the amount of instructions allowed in "all" traces.',
+    'vec_ratio': 'an integer (0-10 transfored into a float by X / 10.0) statements that have vector equivalents '
                  'divided by the total number of trace instructions.',
 }
 
@@ -577,8 +577,8 @@
               'vec': 0,
               'vec_all': 0,
               'vec_cost': 0,
-              'vec_length': 50,
-              'vec_ratio': 6,
+              'vec_length': 60,
+              'vec_ratio': 2,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 

From noreply at buildbot.pypy.org  Thu Aug 20 09:50:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 09:50:43 +0200 (CEST)
Subject: [pypy-commit] pypy default: Fix if the pypy root dir is not in
	PYTHONPATH at all
Message-ID: <20150820075043.A2EBF1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79081:ba4a1979ed5b
Date: 2015-08-20 09:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ba4a1979ed5b/

Log:	Fix if the pypy root dir is not in PYTHONPATH at all

diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,6 @@
 import time, os, sys
+if __name__ == '__main__':
+    sys.path += ['../../../..']    # for subprocess in test_interpreted
 import py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
@@ -58,8 +60,8 @@
 def test_interpreted():
     # takes forever if the Python process is already big...
     import subprocess
-    env = os.environ.copy()
-    subprocess.check_call([sys.executable, __file__],env=env)
+    subprocess.check_call([sys.executable, os.path.basename(__file__)],
+                          cwd=(os.path.dirname(__file__) or '.'))
 
 def test_compiled():
     fn = compile(main, [], gcpolicy="minimark")

From noreply at buildbot.pypy.org  Thu Aug 20 11:09:43 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 20 Aug 2015 11:09:43 +0200 (CEST)
Subject: [pypy-commit] pypy value-profiling: fix problem when the class is
	not a constant
Message-ID: <20150820090943.991F61C05CE@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: value-profiling
Changeset: r79084:ab5fe2fb0615
Date: 2015-08-20 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/ab5fe2fb0615/

Log:	fix problem when the class is not a constant

diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -335,8 +335,9 @@
         value = self.getvalue(op.getarg(0))
         expectedclassbox = op.getarg(1)
         if not isinstance(expectedclassbox, Const):
-            value.make_nonnull(None)
             # can't optimize
+            if not value.is_nonnull():
+                value.make_nonnull(None)
             return
         realclassbox = value.get_constant_class(self.optimizer.cpu)
         if realclassbox is not None:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7116,6 +7116,21 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [p0, i0]
+        p1 = getfield_gc(p0, descr=nextdescr)
+        guard_class(p1, ConstClass(node_vtable)) []
+        record_exact_class(p1, i0)
+        jump(p1, i0)
+        """
+        expected = """
+        [p0, i0]
+        p1 = getfield_gc(p0, descr=nextdescr)
+        guard_class(p1, ConstClass(node_vtable)) []
+        jump(p1, i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_quasi_immut(self):
         ops = """
         [p0, p1, i0]

From noreply at buildbot.pypy.org  Thu Aug 20 11:58:33 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 11:58:33 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-updated-backend: a few fixes
Message-ID: <20150820095833.9DA351C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: ppc-updated-backend
Changeset: r79085:a3eacc21865e
Date: 2015-08-20 02:57 -0700
http://bitbucket.org/pypy/pypy/changeset/a3eacc21865e/

Log:	a few fixes

diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -951,10 +951,9 @@
             self.cb.insts[i + startindex] = new_inst
 
 class PPCBuilder(BlockBuilderMixin, PPCAssembler):
-    def __init__(self, failargs_limit=1000, r0_in_use=False):
+    def __init__(self):
         PPCAssembler.__init__(self)
         self.init_block_builder()
-        self.r0_in_use = r0_in_use
         self.ops_offset = {}
 
     def mark_op(self, op):
@@ -1214,12 +1213,14 @@
                     self.cmpld(block, a, b)
                 
     def alloc_scratch_reg(self):
-        assert not self.r0_in_use
-        self.r0_in_use = True
+        pass
+        #assert not self.r0_in_use
+        #self.r0_in_use = True
 
     def free_scratch_reg(self):
-        assert self.r0_in_use
-        self.r0_in_use = False
+        pass
+        #assert self.r0_in_use
+        #self.r0_in_use = False
 
 class scratch_reg(object):
     def __init__(self, mc):
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -437,7 +437,7 @@
         mc.b_abs(self.propagate_exception_path)
 
         mc.prepare_insts_blocks()
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         # here we do not need a function descr. This is being only called using
         # an internal ABI
         self.malloc_slowpath = rawstart
@@ -539,7 +539,7 @@
         mc.blr()
 
         mc.prepare_insts_blocks()
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         if IS_PPC_64:
             self.write_64_bit_func_descr(rawstart, rawstart+3*WORD)
         self.stack_check_slowpath = rawstart
@@ -597,7 +597,7 @@
         mc.blr()
         #
         mc.prepare_insts_blocks()
-        rawstart = mc.materialize(self.cpu.asmmemmgr, [])
+        rawstart = mc.materialize(self.cpu, [])
         self.wb_slowpath[withcards + 2 * withfloats] = rawstart
 
     def _build_propagate_exception_path(self):
@@ -611,7 +611,7 @@
         mc.load_imm(r.RES, self.cpu.propagate_exception_descr)
         self._gen_epilogue(mc)
         mc.prepare_insts_blocks()
-        self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, [])
+        self.propagate_exception_path = mc.materialize(self.cpu, [])
 
     # The code generated here serves as an exit stub from
     # the executed machine code.
@@ -642,8 +642,7 @@
         self._gen_epilogue(mc)
 
         mc.prepare_insts_blocks()
-        return mc.materialize(self.cpu.asmmemmgr, [],
-                                   self.cpu.gc_ll_descr.gcrootmap)
+        return mc.materialize(self.cpu, [], self.cpu.gc_ll_descr.gcrootmap)
 
     def _gen_epilogue(self, mc):
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
@@ -1115,7 +1114,7 @@
         self.datablockwrapper.done()
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
-        start = self.mc.materialize(self.cpu.asmmemmgr, allblocks, 
+        start = self.mc.materialize(self.cpu, allblocks,
                                     self.cpu.gc_ll_descr.gcrootmap)
         #from pypy.rlib.rarithmetic import r_uint
         #print "=== Loop start is at %s ===" % hex(r_uint(start))
diff --git a/rpython/jit/backend/ppc/test/test_runner.py b/rpython/jit/backend/ppc/test/test_runner.py
--- a/rpython/jit/backend/ppc/test/test_runner.py
+++ b/rpython/jit/backend/ppc/test/test_runner.py
@@ -9,8 +9,7 @@
                                             ConstInt, ConstPtr,
                                             Const,
                                             BoxFloat, ConstFloat)
-from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rffi, rclass
-from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.jit.backend.ppc.arch import IS_PPC_32
 import py

From noreply at buildbot.pypy.org  Thu Aug 20 12:01:39 2015
From: noreply at buildbot.pypy.org (felixonmars)
Date: Thu, 20 Aug 2015 12:01:39 +0200 (CEST)
Subject: [pypy-commit] cffi fix-zdist-test: fix python 3.x compatibility for
	test_zdist
Message-ID: <20150820100139.E114E1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Felix Yan 
Branch: fix-zdist-test
Changeset: r2249:7a380048d8c6
Date: 2015-08-20 16:24 +0800
http://bitbucket.org/cffi/cffi/changeset/7a380048d8c6/

Log:	fix python 3.x compatibility for test_zdist

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -320,19 +320,20 @@
                 ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
                 from distutils.command.build_ext import build_ext
                 import os
 
                 class TestBuildExt(build_ext):
                     def pre_run(self, ext, ffi):
-                        print '_make_setuptools_api: in pre_run:',
+                        print('_make_setuptools_api: in pre_run:', end=" ")
                         assert ffi._hi_there == 42
                         assert ext.name == "pack3.mymod"
                         fn = os.path.join(os.path.dirname(self.build_lib),
                                           '..', 'see_me')
-                        print 'creating %r' % (fn,)
+                        print('creating %r' % (fn,))
                         open(fn, 'w').close()
 
                 setup(name='example1',

From noreply at buildbot.pypy.org  Thu Aug 20 12:01:41 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 12:01:41 +0200 (CEST)
Subject: [pypy-commit] cffi default: Merged in
 felixonmars/cffi/fix-zdist-test (pull request #63)
Message-ID: <20150820100141.DB0E11C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2250:c830b79802de
Date: 2015-08-20 12:02 +0200
http://bitbucket.org/cffi/cffi/changeset/c830b79802de/

Log:	Merged in felixonmars/cffi/fix-zdist-test (pull request #63)

	Fix python 3.x compatibility for test_zdist

diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -320,19 +320,20 @@
                 ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
                 from distutils.command.build_ext import build_ext
                 import os
 
                 class TestBuildExt(build_ext):
                     def pre_run(self, ext, ffi):
-                        print '_make_setuptools_api: in pre_run:',
+                        print('_make_setuptools_api: in pre_run:', end=" ")
                         assert ffi._hi_there == 42
                         assert ext.name == "pack3.mymod"
                         fn = os.path.join(os.path.dirname(self.build_lib),
                                           '..', 'see_me')
-                        print 'creating %r' % (fn,)
+                        print('creating %r' % (fn,))
                         open(fn, 'w').close()
 
                 setup(name='example1',

From noreply at buildbot.pypy.org  Thu Aug 20 12:11:57 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 12:11:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: Document these two branches
Message-ID: <20150820101157.5A21F1C0933@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79086:32f5175b444e
Date: 2015-08-20 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/32f5175b444e/

Log:	Document these two branches

diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -32,7 +32,10 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+Part of cffi 1.2.
+
 .. branch: cffi-new-allocator
+Part of cffi 1.2.
 
 .. branch: unicode-dtype
 

From noreply at buildbot.pypy.org  Thu Aug 20 13:44:16 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 20 Aug 2015 13:44:16 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: accum problem with saving data on
 fail descr (work in progress)
Message-ID: <20150820114416.2BF3A1C05CE@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79087:a9ccb0f0842b
Date: 2015-08-20 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/a9ccb0f0842b/

Log:	accum problem with saving data on fail descr (work in progress)

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -322,6 +322,7 @@
             if accum:
                 loc = self.loc(accum.getoriginalbox())
                 faillocs.append(loc)
+                import pdb; pdb.set_trace()
                 self.update_accumulation_loc(arg, accum, descr, i)
             else:
                 faillocs.append(self.loc(arg))
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -220,6 +220,27 @@
         res = self.meta_interp(f, [60,58.4547])
         assert res == f(60,58.4547) == 58.4547
 
+    def test_accum(self):
+        myjitdriver = JitDriver(greens = [], reds = 'auto')
+        T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
+        def f(d, value):
+            va = lltype.malloc(T, d, flavor='raw', zero=True)
+            for i in range(d):
+                va[i] = value
+            r = 0
+            i = 0
+            while i < d:
+                myjitdriver.jit_merge_point()
+                if i >= d:
+                    raise IndexError
+                r += va[i]
+                i += 1
+            lltype.free(va, flavor='raw')
+            return r
+        res = self.meta_interp(f, [60,0.1], vec_all=True)
+        assert res == f(60,0.1) == 60*0.1
+
+
     @py.test.mark.parametrize('i',[15])
     def test_array_bounds_check_elimination(self,i):
         myjitdriver = JitDriver(greens = [],

From noreply at buildbot.pypy.org  Thu Aug 20 18:10:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Aug 2015 18:10:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start passing virtualstate
	tests
Message-ID: <20150820161041.24DE21C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79088:f107bebc383e
Date: 2015-08-20 18:10 +0200
http://bitbucket.org/pypy/pypy/changeset/f107bebc383e/

Log:	start passing virtualstate tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -117,6 +117,7 @@
     #nodebox = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node))
     node2 = lltype.malloc(NODE2)
     node2.parent.parent.typeptr = node_vtable2
+    node2addr = lltype.cast_opaque_ptr(llmemory.GCREF, node2)
     myptr = lltype.cast_opaque_ptr(llmemory.GCREF, node)
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -25,21 +25,6 @@
         self.optearlyforce = None
 
 class BaseTestGenerateGuards(BaseTest):
-
-    def _box_or_value(self, boxinfo=None):
-        if boxinfo is None:
-            return None, None
-        else:
-            xxx
-        # el
-        if isinstance(box_or_value, OptValue):
-            value = box_or_value
-            box = value.box
-        else:
-            box = box_or_value
-            value = OptValue(box)
-        return value, box
-
     def guards(self, info1, info2, box_or_value, expected, inputargs=None):
         value, box = self._box_or_value(box_or_value)
         if inputargs is None:
@@ -61,22 +46,20 @@
         assert equaloplists(guards, loop.operations, False,
                             boxmap)
 
-    def check_no_guards(self, info1, info2, boxinfo=None, state=None):
-        assert boxinfo is None
-        boxinfo, _ = self._box_or_value(boxinfo)
+    def check_no_guards(self, info1, info2, box=None, opinfo=None, state=None):
         if info1.position == -1:
             info1.position = 0
         if info2.position == -1:
             info2.position = 0
         if state is None:
             state = GenerateGuardState(self.cpu)
-        info1.generate_guards(info2, boxinfo, None, state)
+        info1.generate_guards(info2, box, opinfo, state)
         assert not state.extra_guards
         return state
 
     def check_invalid(self, info1, info2, box_or_value=None, state=None):
         assert box_or_value is None
-        value, _ = self._box_or_value(box_or_value)
+        value, _ = None, None # self._box_or_value(box_or_value)
         if info1.position == -1:
             info1.position = 0
         if info2.position == -1:
@@ -175,8 +158,8 @@
         assert not isgeneral('r', value1, 'r', value2)
 
     def test_field_matching_generalization(self):
-        const1 = NotVirtualStateInfo(OptValue(ConstInt(1)))
-        const2 = NotVirtualStateInfo(OptValue(ConstInt(2)))
+        const1 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(1))
+        const2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(2))
         const1.position = const2.position = 1
         self.check_invalid(const1, const2)
         self.check_invalid(const2, const1)
@@ -197,20 +180,17 @@
         fldtst(VArrayStructStateInfo(fakedescr, [[fielddescr]]), VArrayStructStateInfo(fakedescr, [[fielddescr]]))
 
     def test_known_class_generalization(self):
-        knownclass1 = PtrOptValue(BoxPtr())
-        knownclass1.make_constant_class(None, ConstPtr(self.myptr))
-        info1 = NotVirtualStateInfo(knownclass1)
+        knownclass1 = info.InstancePtrInfo(ConstPtr(self.myptr))
+        info1 = NotVirtualStateInfo(self.cpu, 'r', knownclass1)
         info1.position = 0
-        knownclass2 = PtrOptValue(BoxPtr())
-        knownclass2.make_constant_class(None, ConstPtr(self.myptr))
-        info2 = NotVirtualStateInfo(knownclass2)
+        knownclass2 = info.InstancePtrInfo(ConstPtr(self.myptr))
+        info2 = NotVirtualStateInfo(self.cpu, 'r', knownclass2)
         info2.position = 0
         self.check_no_guards(info1, info2)
         self.check_no_guards(info2, info1)
 
-        knownclass3 = PtrOptValue(BoxPtr())
-        knownclass3.make_constant_class(None, ConstPtr(self.myptr2))
-        info3 = NotVirtualStateInfo(knownclass3)
+        knownclass3 = info.InstancePtrInfo(ConstPtr(self.myptr2))
+        info3 = NotVirtualStateInfo(self.cpu, 'r', knownclass3)
         info3.position = 0
         self.check_invalid(info1, info3)
         self.check_invalid(info2, info3)
@@ -228,49 +208,46 @@
 
     def test_generate_guards_nonvirtual_all_combinations(self):
         # set up infos
-        unknown_val = PtrOptValue(self.nodebox)
-        unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
-        unknown_info = NotVirtualStateInfo(unknown_val)
+        #unknown_val = PtrOptValue(self.nodebox)
+        #unknownnull_val = PtrOptValue(BoxPtr(self.nullptr))
+        unknown_info = NotVirtualStateInfo(self.cpu, 'r', None)
 
-        nonnull_val = PtrOptValue(self.nodebox)
-        nonnull_val.make_nonnull(None)
-        nonnull_info = NotVirtualStateInfo(nonnull_val)
+        nonnull_info = NotVirtualStateInfo(self.cpu, 'r', info.NonNullPtrInfo())
 
-        knownclass_val = PtrOptValue(self.nodebox)
-        classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        knownclass_val.make_constant_class(None, classbox,)
-        knownclass_info = NotVirtualStateInfo(knownclass_val)
-        knownclass2_val = PtrOptValue(self.nodebox2)
-        classbox = self.cpu.ts.cls_of_box(self.nodebox2)
-        knownclass2_val.make_constant_class(None, classbox)
-        knownclass2_info = NotVirtualStateInfo(knownclass2_val)
+        classbox1 = self.cpu.ts.cls_of_box(ConstPtr(self.nodeaddr))
+        knownclass_info = NotVirtualStateInfo(self.cpu, 'r',
+                                              info.InstancePtrInfo(classbox1))
+        classbox2 = self.cpu.ts.cls_of_box(ConstPtr(self.node2addr))
+        knownclass2_info = NotVirtualStateInfo(self.cpu, 'r',
+                                               info.InstancePtrInfo(classbox2))
 
-        constant_val = IntOptValue(BoxInt())
-        constant_val.make_constant(ConstInt(1))
-        constant_info = NotVirtualStateInfo(constant_val)
-        constclass_val = PtrOptValue(self.nodebox)
-        constclass_val.make_constant(self.nodebox.constbox())
-        constclass_info = NotVirtualStateInfo(constclass_val)
-        constclass2_val = PtrOptValue(self.nodebox)
-        constclass2_val.make_constant(self.nodebox2.constbox())
-        constclass2_info = NotVirtualStateInfo(constclass2_val)
-        constantnull_val = PtrOptValue(ConstPtr(self.nullptr))
-        constantnull_info = NotVirtualStateInfo(constantnull_val)
+        constant_info = NotVirtualStateInfo(self.cpu, 'i',
+                                            ConstIntBound(1))
+        constclass_val = info.ConstPtrInfo(ConstPtr(self.nodeaddr))
+        constclass_info = NotVirtualStateInfo(self.cpu, 'r', constclass_val)
+        constclass2_info = NotVirtualStateInfo(self.cpu, 'r',
+                    info.ConstPtrInfo(ConstPtr(self.node2addr)))
+        constantnull_info = NotVirtualStateInfo(self.cpu, 'r',
+                    info.ConstPtrInfo(ConstPtr(self.nullptr)))
 
         # unknown unknown
-        self.check_no_guards(unknown_info, unknown_info, unknown_val)
         self.check_no_guards(unknown_info, unknown_info)
+        self.check_no_guards(unknown_info, unknown_info,
+                             InputArgRef(), info.PtrInfo())
 
         # unknown nonnull
-        self.check_no_guards(unknown_info, nonnull_info, nonnull_val)
+        self.check_no_guards(unknown_info, nonnull_info,
+                             InputArgRef(), info.NonNullPtrInfo())
         self.check_no_guards(unknown_info, nonnull_info)
 
         # unknown knownclass
-        self.check_no_guards(unknown_info, knownclass_info, knownclass_val)
+        self.check_no_guards(unknown_info, knownclass_info,
+                             InputArgRef(), info.InstancePtrInfo(classbox1))
         self.check_no_guards(unknown_info, knownclass_info)
 
         # unknown constant
-        self.check_no_guards(unknown_info, constant_info, constant_val)
+        self.check_no_guards(unknown_info, constant_info,
+                             ConstInt(1), ConstIntBound(1))
         self.check_no_guards(unknown_info, constant_info)
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -122,14 +122,17 @@
             raise VirtualStatesCantMatch("field descrs don't match")
 
         for i in range(len(self.fielddescrs)):
-            yyy
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 raise VirtualStatesCantMatch("field descrs don't match")
-            if value is not None:
-                v = value._fields[self.fielddescrs[i]] # must be there
+            if box is not None:
+                fieldbox = opinfo._fields[self.fielddescrs[i].get_index()]
+                # must be there
+                fieldinfo = fieldbox.get_forwarded()
             else:
-                v = None
-            self.fieldstate[i].generate_guards(other.fieldstate[i], v, state)
+                fieldbox = None
+                fieldinfo = None
+            self.fieldstate[i].generate_guards(other.fieldstate[i], fieldbox,
+                                               fieldinfo, state)
 
 
     def _generalization_of_structpart(self, other):
@@ -196,14 +199,15 @@
             raise VirtualStatesCantMatch("other is a different kind of array")
         if len(self.fieldstate) != len(other.fieldstate):
             raise VirtualStatesCantMatch("other has a different length")
-        v = None
+        fieldbox = None
+        fieldinfo = None
         for i in range(len(self.fieldstate)):
-            xxxx
-            if value is not None:
-                assert isinstance(value, virtualize.VArrayValue)
-                v = value._items[i]
+            if box is not None:
+                assert isinstance(opinfo, info.ArrayPtrInfo)
+                fieldbox = opinfo._items[i]
+                fieldinfo = fieldbox.get_forwarded()
             self.fieldstate[i].generate_guards(other.fieldstate[i],
-                                               v, state)
+                                            fieldbox, fieldinfo, state)
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
@@ -234,7 +238,7 @@
         self.arraydescr = arraydescr
         self.fielddescrs = fielddescrs
 
-    def _generate_guards(self, other, value, state):
+    def _generate_guards(self, other, box, opinfo, state):
         if not isinstance(other, VArrayStructStateInfo):
             raise VirtualStatesCantMatch("other is not an VArrayStructStateInfo")
         if self.arraydescr is not other.arraydescr:
@@ -244,7 +248,8 @@
             raise VirtualStatesCantMatch("other has a different length")
 
         p = 0
-        v = None
+        fieldbox = None
+        fieldinfo = None
         for i in range(len(self.fielddescrs)):
             if len(self.fielddescrs[i]) != len(other.fielddescrs[i]):
                 raise VirtualStatesCantMatch("other has a different length")
@@ -252,11 +257,12 @@
                 descr = self.fielddescrs[i][j]
                 if descr is not other.fielddescrs[i][j]:
                     raise VirtualStatesCantMatch("other is a different kind of array")
-                if value is not None:
+                if box is not None:
+                    xxx
                     assert isinstance(value, virtualize.VArrayStructValue)
                     v = value._items[i][descr]
                 self.fieldstate[p].generate_guards(other.fieldstate[p],
-                                                   v,
+                                                   fieldbox, fieldinfo,
                                                    state)
                 p += 1
 

From noreply at buildbot.pypy.org  Thu Aug 20 19:29:02 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 20 Aug 2015 19:29:02 +0200 (CEST)
Subject: [pypy-commit] pypy default: add a check
Message-ID: <20150820172902.D208C1C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r79089:592ce9777d88
Date: 2015-08-20 19:29 +0200
http://bitbucket.org/pypy/pypy/changeset/592ce9777d88/

Log:	add a check

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -268,6 +268,7 @@
                 y -= 1
             return res
         res = self.meta_interp(f, [6, sys.maxint, 48])
+        self.check_trace_count(6)
         assert res == f(6, sys.maxint, 48)
 
     def test_loop_invariant_mul_bridge_ovf2(self):

From noreply at buildbot.pypy.org  Thu Aug 20 23:09:07 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 23:09:07 +0200 (CEST)
Subject: [pypy-commit] cffi default: Release 1.2.1
Message-ID: <20150820210907.366D71C0063@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2251:55ac8bf788d0
Date: 2015-08-20 23:09 +0200
http://bitbucket.org/cffi/cffi/changeset/55ac8bf788d0/

Log:	Release 1.2.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6298,7 +6298,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.2.0");
+    v = PyText_FromString("1.2.1");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3438,4 +3438,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.2'
 # The full version, including alpha/beta/rc tags.
-release = '1.2.0'
+release = '1.2.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,11 +51,11 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0-1.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz
 
-   - MD5: 092a40d5e8e8f71742008e28e2b2f803
+   - MD5: ...
 
-   - SHA: 496a9f094b86e09d8a27255f850d7fe3b5115038
+   - SHA: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list `_
 """,
-        version='1.2.0',
+        version='1.2.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},

From noreply at buildbot.pypy.org  Thu Aug 20 23:09:22 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 23:09:22 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: hg merge default
Message-ID: <20150820210922.872111C0063@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-1.2
Changeset: r2252:0cf51149cdd8
Date: 2015-08-20 23:10 +0200
http://bitbucket.org/cffi/cffi/changeset/0cf51149cdd8/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6298,7 +6298,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.2.0");
+    v = PyText_FromString("1.2.1");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3438,4 +3438,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.2'
 # The full version, including alpha/beta/rc tags.
-release = '1.2.0'
+release = '1.2.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,11 +51,11 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.0-1.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz
 
-   - MD5: 092a40d5e8e8f71742008e28e2b2f803
+   - MD5: ...
 
-   - SHA: 496a9f094b86e09d8a27255f850d7fe3b5115038
+   - SHA: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list `_
 """,
-        version='1.2.0',
+        version='1.2.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -320,19 +320,20 @@
                 ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
                 from distutils.command.build_ext import build_ext
                 import os
 
                 class TestBuildExt(build_ext):
                     def pre_run(self, ext, ffi):
-                        print '_make_setuptools_api: in pre_run:',
+                        print('_make_setuptools_api: in pre_run:', end=" ")
                         assert ffi._hi_there == 42
                         assert ext.name == "pack3.mymod"
                         fn = os.path.join(os.path.dirname(self.build_lib),
                                           '..', 'see_me')
-                        print 'creating %r' % (fn,)
+                        print('creating %r' % (fn,))
                         open(fn, 'w').close()
 
                 setup(name='example1',

From noreply at buildbot.pypy.org  Thu Aug 20 23:11:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 20 Aug 2015 23:11:53 +0200 (CEST)
Subject: [pypy-commit] cffi release-1.2: md5/sha1
Message-ID: <20150820211153.07C681C0063@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-1.2
Changeset: r2253:953d7df27924
Date: 2015-08-20 23:12 +0200
http://bitbucket.org/cffi/cffi/changeset/953d7df27924/

Log:	md5/sha1

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,9 +53,9 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz
 
-   - MD5: ...
+   - MD5: faca1e420e80433db409bb1bdd0a3f45
 
-   - SHA: ...
+   - SHA: f7ed014ff1602a8e81073f5356b4cafbc3f5dce1
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From noreply at buildbot.pypy.org  Fri Aug 21 09:13:21 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 21 Aug 2015 09:13:21 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: flesh out release notice
Message-ID: <20150821071321.91DFE1C0063@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79090:15760ef18145
Date: 2015-08-21 10:13 +0300
http://bitbucket.org/pypy/pypy/changeset/15760ef18145/

Log:	flesh out release notice

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -4,9 +4,8 @@
 
 We're pleased to announce PyPy 2.6.1, an update to PyPy 2.6.0 released June 1.
 We have updated stdlib to 2.7.10, `cffi`_ to version 1.2, extended support for
-the new vmprof_ statistical profiler for multiple threads, partially
-implemented unicode dtypes in numpy, added support for the axis keyword to
-many numpy methods, and in general improved numpy compatability.
+the new vmprof_ statistical profiler for multiple threads, and increased
+functionality of numpy.
 
 You can download the PyPy 2.6.1 release here:
 
@@ -72,9 +71,13 @@
 
 * Bug Fixes
 
-  * 
+  * Revive non-SSE2 support
 
-  * 
+  * Fixes for detaching _io.Buffer*
+
+  * On Windows, close (and flush) all open sockets on exiting
+
+  * Drop support for ancient macOS v10.4 and before
 
   * Issues reported with our previous release were resolved_ after reports from users on
     our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
@@ -82,23 +85,54 @@
 
 * New features:
 
-  * 
+  * cffi was updated to version 1.2
+
+  * The python stdlib was updated to 2.7.10 from 2.7.9
+
+  * vmprof now supports multiple threads
+
+  * The translation process builds cffi import libraries for some stdlib
+    packages, which should prevent confusion when package.py is not used
+
+  * better support for gdb debugging
+
+  * freebsd should be able to translate PyPy "out of the box" with no patches
 
 * Numpy:
 
-  * 
+  * Better support for record dtypes, including the ``align`` keyword
 
-  * 
+  * Implement casting and create output arrays accordingly (still missing some corner cases)
+
+  * Support creation of unicode ndarrays
+
+  * Better support ndarray.flags
+
+  * Support ``axis`` argument in more functions
+
+  * Refactor array indexing to support ellipses
+
+  * Allow the docstrings of built-in numpy objects to be set at run-time
+
+  * Support the ``buffered`` nditer creation keyword
 
 * Performance improvements:
 
-  * 
+  * Delay recursive calls to make them non-recursive
 
-  * 
+  * Skip loop unrolling if it compiles too much code
 
-  * 
+  * Tweak the heapcache
 
-  * 
+  * Add a list strategy for lists that store both floats and 32-bit integers.
+    The latter are encoded as nonstandard NaNs.  Benchmarks show that the speed
+    of such lists is now very close to the speed of purely-int or purely-float
+    lists. 
+
+  * Simplify implementation of ffi.gc() to avoid most weakrefs
+
+  * Massively improve the performance of map() with more than
+    one sequence argument
 
 .. _`vmprof`: https://vmprof.readthedocs.org
 .. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.0.html

From noreply at buildbot.pypy.org  Fri Aug 21 10:28:26 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 10:28:26 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: llgraph is now working with accum
	data
Message-ID: <20150821082826.AD58F1C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79091:10582848a63f
Date: 2015-08-20 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/10582848a63f/

Log:	llgraph is now working with accum data

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -265,8 +265,8 @@
         self.stats = stats or MiniStats()
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
-    def stitch_bridge(self, faildescr, jitcell_token):
-        pass
+    def stitch_bridge(self, faildescr, compiled_faildescr, jitcell_token):
+        faildescr._llgraph_bridge = compiled_faildescr._llgraph_bridge
 
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
                      unique_id=0, log=True, name='', logger=None):
@@ -736,7 +736,10 @@
     def bh_vec_box_unpack(self, vx, index):
         return vx[index]
 
-    def bh_vec_expand(self, x, count):
+    def bh_vec_float_expand(self, x, count):
+        return [x] * count
+
+    def bh_vec_int_expand(self, x, count):
         return [x] * count
 
     def bh_vec_int_signext(self, vx, ext):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -70,7 +70,7 @@
         self.rtyper = cpu.rtyper
         self._debug = False
 
-    def stitch_bridge(self, faildescr, token):
+    def stitch_bridge(self, faildescr, compiled_faildescr, token):
         raise NotImplementedError
 
     def setup_once(self):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -97,8 +97,8 @@
                                             inputargs, operations,
                                             looptoken, log=log)
 
-    def stitch_bridge(self, faildescr, token):
-        self.assembler.stitch_bridge(faildescr, token)
+    def stitch_bridge(self, faildescr, compiled_faildescr, token):
+        self.assembler.stitch_bridge(faildescr, compiled_faildescr, token)
 
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -505,7 +505,6 @@
         full_size = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(looptoken)
-        looptoken.rawstart = rawstart
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         looptoken._ll_loop_code = looppos + rawstart
@@ -570,6 +569,7 @@
         fullsize = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(original_loop_token)
+        faildescr.rawstart = rawstart
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         debug_bridge(descr_number, rawstart, codeendpos)
@@ -592,9 +592,8 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
-    def stitch_bridge(self, faildescr, token):
-        assert token.rawstart != 0
-        self.patch_jump_for_descr(faildescr, token.rawstart)
+    def stitch_bridge(self, faildescr, compiled_faildescr, token):
+        self.patch_jump_for_descr(faildescr, compiled_faildescr.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -186,7 +186,8 @@
 
 def generate_pending_loop_versions(loop, jitdriver_sd, metainterp, jitcell_token):
     """ if a loop version is created for a guard instruction (e.g. they are known
-        to fail frequently, then a version can be created that is immediatly compiled.
+        to fail frequently) then a version can be created that is immediatly compiled
+        and stitched to the guard.
     """
     metainterp_sd = metainterp.staticdata
     cpu = metainterp_sd.cpu
@@ -203,8 +204,8 @@
                                    faildescr, version.inputargs,
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
-            for faildescr in version.faildescrs[1:]:
-                cpu.stitch_bridge(faildescr, jitcell_token)
+            for fd in version.faildescrs[1:]:
+                cpu.stitch_bridge(fd, faildescr, jitcell_token)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -515,7 +515,7 @@
             i += 1
         else:
             expand_opnum = getexpandopnum(box_type)
-            op = ResOperation(expand_opnum, [arg], vbox)
+            op = ResOperation(expand_opnum, [arg, ConstInt(vbox.item_count)], vbox)
             ops.append(op)
             if variables is not None:
                 variables.append(vbox)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -528,8 +528,8 @@
     'VEC_FLOAT_PACK/4',          # VEC_FLOAT_PACK(vX, var/const, index, item_count)
     'VEC_INT_UNPACK/3',          # iX|fX = VEC_INT_UNPACK(vX, index, item_count)
     'VEC_INT_PACK/4',            # VEC_INT_PACK(vX, var/const, index, item_count)
-    'VEC_FLOAT_EXPAND/1',        # vX = VEC_FLOAT_EXPAND(var/const)
-    'VEC_INT_EXPAND/1',          # vX = VEC_INT_EXPAND(var/const)
+    'VEC_FLOAT_EXPAND/2',        # vX = VEC_FLOAT_EXPAND(var/const, item_count)
+    'VEC_INT_EXPAND/2',          # vX = VEC_INT_EXPAND(var/const, item_count)
     'VEC_BOX/1',
     '_VEC_PURE_LAST',
     #
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -194,7 +194,7 @@
             i = 0
             while i < d:
                 myjitdriver.jit_merge_point()
-                if not (i < 60):
+                if not (i < d):
                     raise IndexError
                 va[i] = va[i] + 34.5
                 i += 1
@@ -231,14 +231,14 @@
             i = 0
             while i < d:
                 myjitdriver.jit_merge_point()
-                if i >= d:
+                if not(i < d):
                     raise IndexError
                 r += va[i]
                 i += 1
             lltype.free(va, flavor='raw')
             return r
-        res = self.meta_interp(f, [60,0.1], vec_all=True)
-        assert res == f(60,0.1) == 60*0.1
+        res = self.meta_interp(f, [60,0.5], vec_all=True)
+        assert res == f(60,0.5) == 60*0.5
 
 
     @py.test.mark.parametrize('i',[15])

From noreply at buildbot.pypy.org  Fri Aug 21 10:28:28 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 10:28:28 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: moved the collection of compile
 loop version descr to guard opt. the assembler writes the correct offset
 into memory but it is not visible in the resulting memory block (e.g.
 jitviewer?). something is wrong
Message-ID: <20150821082828.D91DD1C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79092:27ddd5637fb8
Date: 2015-08-21 10:28 +0200
http://bitbucket.org/pypy/pypy/changeset/27ddd5637fb8/

Log:	moved the collection of compile loop version descr to guard opt. the
	assembler writes the correct offset into memory but it is not
	visible in the resulting memory block (e.g. jitviewer?). something
	is wrong

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -265,8 +265,8 @@
         self.stats = stats or MiniStats()
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
-    def stitch_bridge(self, faildescr, compiled_faildescr, jitcell_token):
-        faildescr._llgraph_bridge = compiled_faildescr._llgraph_bridge
+    def stitch_bridge(self, faildescr, target):
+        faildescr._llgraph_bridge = target
 
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
                      unique_id=0, log=True, name='', logger=None):
@@ -285,6 +285,7 @@
         faildescr._llgraph_bridge = lltrace
         clt._llgraph_alltraces.append(lltrace)
         self._record_labels(lltrace)
+        return lltrace
 
     def _record_labels(self, lltrace):
         for i, op in enumerate(lltrace.operations):
diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -35,7 +35,6 @@
         self.exc = exc
         self.is_guard_not_invalidated = is_guard_not_invalidated
         self.is_guard_not_forced = is_guard_not_forced
-        self.rawstart = 0
 
     def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth):
         # note that regalloc has a very similar compute, but
@@ -70,7 +69,7 @@
         self.rtyper = cpu.rtyper
         self._debug = False
 
-    def stitch_bridge(self, faildescr, compiled_faildescr, token):
+    def stitch_bridge(self, faildescr, target):
         raise NotImplementedError
 
     def setup_once(self):
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -97,8 +97,8 @@
                                             inputargs, operations,
                                             looptoken, log=log)
 
-    def stitch_bridge(self, faildescr, compiled_faildescr, token):
-        self.assembler.stitch_bridge(faildescr, compiled_faildescr, token)
+    def stitch_bridge(self, faildescr, target):
+        self.assembler.stitch_bridge(faildescr, target)
 
     def _setup_frame_realloc(self, translate_support_code):
         FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed],
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -569,7 +569,7 @@
         fullsize = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(original_loop_token)
-        faildescr.rawstart = rawstart
+        faildescr.bridge_rawstart = rawstart
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         debug_bridge(descr_number, rawstart, codeendpos)
@@ -592,8 +592,9 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
-    def stitch_bridge(self, faildescr, compiled_faildescr, token):
-        self.patch_jump_for_descr(faildescr, compiled_faildescr.rawstart)
+    def stitch_bridge(self, faildescr, target):
+        assert target != 0
+        self.patch_jump_for_descr(faildescr, target)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
@@ -604,7 +605,8 @@
                 regalloc.position = tok.position
                 tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc)
             else:
-                self.store_info_on_descr(0, tok)
+                startpos = self.mc.get_relative_pos()
+                self.store_info_on_descr(startpos, tok)
         if WORD == 8 and len(self.pending_memoryerror_trampoline_from) > 0:
             self.error_trampoline_64 = self.generate_propagate_error_64()
 
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -322,7 +322,6 @@
             if accum:
                 loc = self.loc(accum.getoriginalbox())
                 faillocs.append(loc)
-                import pdb; pdb.set_trace()
                 self.update_accumulation_loc(arg, accum, descr, i)
             else:
                 faillocs.append(self.loc(arg))
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -192,6 +192,8 @@
     metainterp_sd = metainterp.staticdata
     cpu = metainterp_sd.cpu
     if loop.versions is not None:
+        # compile each version once for the first fail descr!
+        # this assumes that the root trace (= loop) is already compiled
         for version in loop.versions:
             if len(version.faildescrs) == 0:
                 continue
@@ -200,12 +202,16 @@
             vl.inputargs = version.inputargs
             vl.operations = version.operations
             vl.original_jitcell_token = jitcell_token
-            send_bridge_to_backend(jitdriver_sd, metainterp_sd,
-                                   faildescr, version.inputargs,
-                                   version.operations, jitcell_token)
+            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+                                             faildescr, version.inputargs,
+                                             version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
-            for fd in version.faildescrs[1:]:
-                cpu.stitch_bridge(fd, faildescr, jitcell_token)
+            version.compiled = faildescr.bridge_rawstart
+            assert asminfo is not None
+        # stitch the rest of the traces
+        for version in loop.versions:
+            for faildescr in version.faildescrs[1:]:
+                cpu.stitch_bridge(faildescr, version.compiled)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
@@ -437,6 +443,7 @@
     #if metainterp_sd.warmrunnerdesc is not None:    # for tests
     #    metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(
     #        original_loop_token)
+    return asminfo
 
 # ____________________________________________________________
 
@@ -742,6 +749,11 @@
     def loop_version(self):
         return True
 
+    def clone(self):
+        cloned = ResumeGuardDescr.clone(self)
+        cloned.version = self.version
+        return cloned
+
 class AllVirtuals:
     llopaque = True
     cache = None
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -758,38 +758,7 @@
         label = operations[idx]
         self.label_pos = idx
         self.inputargs = label.getarglist()
-
-    def register_all_guards(self, opt_ops, invariant_arg_count=0):
-        from rpython.jit.metainterp.compile import CompileLoopVersionDescr
-        pass_by = 0
-        idx = index_of_first(rop.LABEL, opt_ops)
-        if opt_ops[idx].getdescr() is not opt_ops[-1].getdescr():
-            idx = index_of_first(rop.LABEL, opt_ops, pass_by=1)
-        assert idx >= 0
-        version_failargs = opt_ops[idx].getarglist()
-        if invariant_arg_count > 0:
-            # constant/variable expansion append arguments to the label
-            # if they are not removed, the register allocator cannot
-            # reconstruct the binding if len(inputargs) != len(faillocs)
-            to = len(version_failargs) - invariant_arg_count
-            assert to >= 0
-            version_failargs = version_failargs[:to]
-
-        for op in opt_ops:
-            if op.is_guard():
-                assert isinstance(op, GuardResOp)
-                descr = op.getdescr()
-                if descr.loop_version():
-                    assert isinstance(descr, CompileLoopVersionDescr)
-                    if descr.version is None:
-                        # currently there is only ONE versioning,
-                        # that is the original loop after unrolling.
-                        # if there are more possibilites, let the descr
-                        # know which loop version he preferes
-                        descr.version = self
-                        self.faildescrs.append(descr)
-                        op.setfailargs(version_failargs)
-                        op.rd_snapshot = None
+        self.compiled = None
 
     def register_guard(self, op):
         from rpython.jit.metainterp.compile import CompileLoopVersionDescr
@@ -798,8 +767,6 @@
         assert isinstance(descr, CompileLoopVersionDescr)
         descr.version = self
         self.faildescrs.append(descr)
-        op.setfailargs(self.inputargs)
-        op.rd_snapshot = None
 
     def update_token(self, jitcell_token, all_target_tokens):
         # this is only invoked for versioned loops!
@@ -892,16 +859,22 @@
         return self.operations
 
     def find_first_index(self, opnum, pass_by=0):
-        """ return the first operation having the same opnum or -1 """
+        """ return the first index of the operation having the same opnum or -1 """
         return index_of_first(opnum, self.operations, pass_by)
 
+    def find_first(self, opnum, pass_by=0):
+        index = self.find_first_index(opnum, pass_by)
+        if index != -1:
+            return self.operations[index]
+        return None
+
     def snapshot(self):
         version = LoopVersion(self.copy_operations())
         self.versions.append(version)
         return version
 
     def copy_operations(self):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
+        from rpython.jit.metainterp.compile import ResumeGuardDescr, CompileLoopVersionDescr
         ignore = (rop.DEBUG_MERGE_POINT,)
         operations = []
         for op in self.operations:
@@ -913,6 +886,8 @@
             if cloned.is_guard() and descr:
                 assert isinstance(descr, ResumeGuardDescr)
                 cloned.setdescr(descr.clone())
+                if isinstance(descr, CompileLoopVersionDescr):
+                    descr.version.register_guard(cloned)
         return operations
 
     def get_display_text(self):    # for graphpage.py
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -102,6 +102,7 @@
         return descr.repr_of_descr()
 
     def repr_of_arg(self, arg):
+        return str(arg) # XXX
         try:
             mv = self.memo[arg]
         except KeyError:
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.jit.metainterp.history import (ConstInt, BoxVector, 
         BoxFloat, BoxInt, ConstFloat, Box, Const)
-from rpython.jit.metainterp.compile import ResumeGuardDescr
+from rpython.jit.metainterp.compile import ResumeGuardDescr, CompileLoopVersionDescr
 from rpython.rlib.objectmodel import we_are_translated
 
 class Guard(object):
@@ -70,7 +70,7 @@
                 return (lc <= 0 and rc >= 0)
         return False
 
-    def transitive_imply(self, other, opt):
+    def transitive_imply(self, other, opt, loop):
         if self.op.getopnum() != other.op.getopnum():
             # stronger restriction, intermixing e.g. <= and < would be possible
             return None
@@ -91,6 +91,8 @@
         assert isinstance(descr, ResumeGuardDescr)
         guard.setdescr(descr.clone())
         guard.setarg(0, box_result)
+        label = loop.find_first(rop.LABEL)
+        guard.setfailargs(label.getarglist())
         opt.emit_operation(guard)
 
         return guard
@@ -149,6 +151,10 @@
     def set_to_none(self, operations):
         assert operations[self.index] is self.op
         operations[self.index] = None
+        descr = self.op.getdescr()
+        if isinstance(descr, CompileLoopVersionDescr) and descr.version:
+            descr.version.faildescrs.remove(descr)
+            descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None
 
@@ -171,12 +177,14 @@
         return Guard(index, guard_op, cmp_op, index_vars)
 
 class GuardStrengthenOpt(object):
-    def __init__(self, index_vars):
+    """ Note that this optimization is only used in the vector optimizer (yet) """
+    def __init__(self, index_vars, has_two_labels):
         self.index_vars = index_vars
         self._newoperations = []
         self.strength_reduced = 0 # how many guards could be removed?
         self.strongest_guards = {}
         self.guards = {}
+        self.has_two_labels = has_two_labels
 
     def collect_guard_information(self, loop):
         operations = loop.operations
@@ -256,8 +264,18 @@
         # the guards are ordered. guards[i] is before guards[j] iff i < j
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
+        #
+        assert len(loop.versions) == 1, "none or more than one version created"
+        version = loop.versions[0]
 
-        if user_code or True:
+        for op in loop.operations:
+            if not op.is_guard():
+                continue
+            descr = op.getdescr()
+            if isinstance(descr, CompileLoopVersionDescr):
+                version.register_guard(op)
+
+        if user_code and False:
             version = loop.snapshot()
             self.eliminate_array_bound_checks(loop, version)
 
@@ -278,11 +296,16 @@
             # iff we add invariant guards
             one = guards[0]
             for other in guards[1:]:
-                transitive_guard = one.transitive_imply(other, self)
+                transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
                     other.set_to_none(loop.operations)
                     version.register_guard(transitive_guard)
 
-        loop.operations = self._newoperations + \
-                [op for op in loop.operations if op]
+        if self.has_two_labels:
+            oplist = [loop.operations[0]] + self._newoperations + \
+                     [op for op in loop.operations[1:] if op]
+            loop.operations = oplist
+        else:
+            loop.operations = self._newoperations + \
+                    [op for op in loop.operations if op]
 
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -856,34 +856,6 @@
         assert not isinstance(box, BoxVector)
         self.box_to_vbox[box] = (off, vector)
 
-    def prepend_invariant_operations(self, oplist, orig_label_args):
-        if len(self.invariant_oplist) > 0:
-            label = oplist[0]
-            assert label.getopnum() == rop.LABEL
-            #
-            jump = oplist[-1]
-            assert jump.getopnum() == rop.JUMP
-            #
-            label_args = label.getarglist()[:]
-            jump_args = jump.getarglist()
-            for var in self.invariant_vector_vars:
-                label_args.append(var)
-                jump_args.append(var)
-            #
-            # in case of any invariant_vector_vars, the label is restored
-            # and the invariant operations are added between the original label
-            # and the new label
-            descr = label.getdescr()
-            assert isinstance(descr, TargetToken)
-            token = TargetToken(descr.targeting_jitcell_token)
-            oplist[0] = label.copy_and_change(label.getopnum(), args=label_args, descr=token)
-            oplist[-1] = jump.copy_and_change(jump.getopnum(), args=jump_args, descr=token)
-            #
-            return [ResOperation(rop.LABEL, orig_label_args, None, descr)] + \
-                   self.invariant_oplist + oplist
-        #
-        return oplist
-
 class Pack(object):
     """ A pack is a set of n statements that are:
         * isomorphic
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_guard.py b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_guard.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
@@ -84,7 +84,7 @@
             if op.is_guard():
                 op.setdescr(compile.CompileLoopVersionDescr())
         dep = DependencyGraph(loop)
-        opt = GuardStrengthenOpt(dep.index_vars)
+        opt = GuardStrengthenOpt(dep.index_vars, False)
         opt.propagate_all_forward(loop, user_code)
         return opt
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -49,10 +49,11 @@
         #
         opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, loop, 0)
         opt.propagate_all_forward()
-        gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
+        #
+        gso = GuardStrengthenOpt(opt.dependency_graph.index_vars, opt.has_two_labels)
         gso.propagate_all_forward(opt.loop, user_code)
         # connect all compile loop version fail descriptors to this version
-        version.register_all_guards(loop.operations, opt.appended_arg_count)
+        #version.register_all_guards(loop.operations, opt.appended_arg_count)
         #
         #
         end = time.clock()
@@ -135,6 +136,7 @@
         self.costmodel = X86_CostModel(cost_threshold, self.cpu.vector_register_size)
         self.appended_arg_count = 0
         self.orig_label_args = None
+        self.has_two_labels = False
 
     def propagate_all_forward(self, clear=True):
         self.clear_newoperations()
@@ -503,11 +505,46 @@
                     accum = arg.getaccum()
                     if accum:
                         accum.save_to_descr(op.getdescr(),i)
-        self.loop.operations = \
-            sched_data.prepend_invariant_operations(self._newoperations,
-                                                    self.orig_label_args)
+            self.has_two_labels = len(sched_data.invariant_oplist) > 0
+            self.loop.operations = self.prepend_invariant_operations(sched_data)
+        else:
+            self.loop.operations = self._newoperations
+        
         self.clear_newoperations()
 
+    def prepend_invariant_operations(self, sched_data):
+        """ Add invariant operations to the trace loop. returns the operation list
+            as first argument and a second a boolean value. it is true if any inva
+        """
+        oplist = self._newoperations
+
+        if len(sched_data.invariant_oplist) > 0:
+            label = oplist[0]
+            assert label.getopnum() == rop.LABEL
+            #
+            jump = oplist[-1]
+            assert jump.getopnum() == rop.JUMP
+            #
+            label_args = label.getarglist()[:]
+            jump_args = jump.getarglist()
+            for var in sched_data.invariant_vector_vars:
+                label_args.append(var)
+                jump_args.append(var)
+            #
+            # in case of any invariant_vector_vars, the label is restored
+            # and the invariant operations are added between the original label
+            # and the new label
+            descr = label.getdescr()
+            assert isinstance(descr, TargetToken)
+            token = TargetToken(descr.targeting_jitcell_token)
+            oplist[0] = label.copy_and_change(label.getopnum(), args=label_args, descr=token)
+            oplist[-1] = jump.copy_and_change(jump.getopnum(), args=jump_args, descr=token)
+            #
+            return [ResOperation(rop.LABEL, self.orig_label_args, None, descr)] + \
+                   sched_data.invariant_oplist + oplist
+        #
+        return oplist
+
     def analyse_index_calculations(self):
         ee_pos = self.loop.find_first_index(rop.GUARD_EARLY_EXIT)
         if len(self.loop.operations) <= 2 or ee_pos == -1:
@@ -580,7 +617,6 @@
             descr.copy_all_attributes_from(olddescr)
         #
         tgt_op.setdescr(descr)
-        tgt_op.rd_snapshot = op.rd_snapshot
         tgt_op.setfailargs(op.getfailargs()[:])
 
 
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -222,17 +222,19 @@
 
     def test_accum(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')
-        T = lltype.Array(rffi.DOUBLE, hints={'nolength': True})
+        T = lltype.Array(rffi.DOUBLE)
         def f(d, value):
             va = lltype.malloc(T, d, flavor='raw', zero=True)
             for i in range(d):
                 va[i] = value
             r = 0
             i = 0
+            k = d + 2
+            # in this case a guard k <= d is inserted which fails right away!
             while i < d:
                 myjitdriver.jit_merge_point()
-                if not(i < d):
-                    raise IndexError
+                #if not(i < k):
+                #    k -= 1
                 r += va[i]
                 i += 1
             lltype.free(va, flavor='raw')

From noreply at buildbot.pypy.org  Fri Aug 21 10:50:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 10:50:43 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-updated-backend: start by copying some more
 logic from x86, and some comments that are more recent than the ppc backend
Message-ID: <20150821085043.C055A1C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: ppc-updated-backend
Changeset: r79093:d289f166a3dc
Date: 2015-08-20 08:35 -0700
http://bitbucket.org/pypy/pypy/changeset/d289f166a3dc/

Log:	start by copying some more logic from x86, and some comments that
	are more recent than the ppc backend

diff --git a/rpython/jit/backend/ppc/arch.py b/rpython/jit/backend/ppc/arch.py
--- a/rpython/jit/backend/ppc/arch.py
+++ b/rpython/jit/backend/ppc/arch.py
@@ -1,50 +1,75 @@
 # Constants that depend on whether we are on 32-bit or 64-bit
 
-from rpython.jit.backend.ppc.register import (NONVOLATILES,
-                                              NONVOLATILES_FLOAT,
-                                              MANAGED_REGS,
-                                              MANAGED_FP_REGS)
+from rpython.jit.backend.ppc import register as r
 
 import sys
 if sys.maxint == (2**31 - 1):
+    assert False, "the ppc backend only supports PPC-64 for now"
     WORD = 4
-    DWORD = 2 * WORD
+    #DWORD = 2 * WORD
     IS_PPC_32 = True
-    BACKCHAIN_SIZE = 2
-    FPR_SAVE_AREA = len(NONVOLATILES_FLOAT) * DWORD
+    #BACKCHAIN_SIZE = 2
+    #FPR_SAVE_AREA = len(NONVOLATILES_FLOAT) * DWORD
 else:
     WORD = 8
-    DWORD = 2 * WORD
+    #DWORD = 2 * WORD
     IS_PPC_32 = False
-    BACKCHAIN_SIZE = 6
-    FPR_SAVE_AREA = len(NONVOLATILES_FLOAT) * WORD
+    #BACKCHAIN_SIZE = 6
+    #FPR_SAVE_AREA = len(NONVOLATILES_FLOAT) * WORD
 
 IS_PPC_64               = not IS_PPC_32
 MY_COPY_OF_REGS         = 0
 
-FORCE_INDEX             = WORD
-GPR_SAVE_AREA           = len(NONVOLATILES) * WORD
-FLOAT_INT_CONVERSION    = WORD
+#FORCE_INDEX             = WORD
+#GPR_SAVE_AREA           = len(NONVOLATILES) * WORD
+#FLOAT_INT_CONVERSION    = WORD
 MAX_REG_PARAMS          = 8
 MAX_FREG_PARAMS         = 13
 # we need at most 5 instructions to load a constant
 # and one instruction to patch the stack pointer
-SIZE_LOAD_IMM_PATCH_SP  = 6
+#SIZE_LOAD_IMM_PATCH_SP  = 6
 
-FORCE_INDEX_OFS         = (len(MANAGED_REGS) + len(MANAGED_FP_REGS)) * WORD
+#FORCE_INDEX_OFS         = (len(MANAGED_REGS) + len(MANAGED_FP_REGS)) * WORD
 
-# The JITFRAME_FIXED_SIZE is measured in words, not bytes or bits.
-# Follwing the PPC ABI, we are saving:
-# - volatile fpr's
-# - volatile gpr's
-# - vrsave word
-# - alignment padding
-# - vector register save area (quadword aligned)
-# 3 + 27 + 1 + 4 + 1
-JITFRAME_FIXED_SIZE = len(MANAGED_FP_REGS) + len(MANAGED_REGS) + 1 + 4 + 1
+
+#
+#        +--------------------+     <-  SP + STD_FRAME_SIZE
+#        |  general registers |
+#        |  save area         |
+#        +--------------------+     <-  SP + 120
+#        |  Local vars (*)    |
+#        +--------------------+     <-  SP + 112
+#        |  Parameter save    |
+#        |  area (8 args max) |
+#        +--------------------+     <-  SP + 48
+#        |  TOC (unused)      |
+#        +--------------------+     <-  SP + 40
+#        |  link ed. (unused) |
+#        +--------------------+     <-  SP + 32
+#        |  compiler (unused) |
+#        +--------------------+     <-  SP + 24
+#        |  LR save area      |
+#        +--------------------+     <-  SP + 16
+#        |  CR save (unused)  |
+#        +--------------------+     <-  SP + 8
+#        |  SP back chain     |
+#        +--------------------+     <-  SP
+
+# The local variables area contains only a copy of the 2nd argument
+# passed to the machine code function, which is the ll_threadlocal_addr.
+# The 1st argument, i.e. the GC-managed jitframe, is stored in the
+# register r31.
+
+
+REGISTERS_SAVED         = [r.r25, r.r26, r.r27, r.r28, r.r29, r.r30, r.r31]
+assert REGISTERS_SAVED == [_r for _r in r.NONVOLATILES
+                              if _r in r.MANAGED_REGS or _r == r.r31]
+
+STD_FRAME_SIZE_IN_BYTES = 120 + len(REGISTERS_SAVED) * WORD
+assert STD_FRAME_SIZE_IN_BYTES % 16 == 0
 
 # offset to LR in BACKCHAIN
 if IS_PPC_32:
     LR_BC_OFFSET = WORD
 else:
-    LR_BC_OFFSET = 2 * WORD
+    LR_BC_OFFSET = 16
diff --git a/rpython/jit/backend/ppc/assembler.py b/rpython/jit/backend/ppc/assembler.py
--- a/rpython/jit/backend/ppc/assembler.py
+++ b/rpython/jit/backend/ppc/assembler.py
@@ -1,6 +1,6 @@
 import os
 from rpython.jit.backend.ppc import form
-from rpython.jit.backend.ppc import asmfunc
+#from rpython.jit.backend.ppc import asmfunc
 
 # don't be fooled by the fact that there's some separation between a
 # generic assembler class and a PPC assembler class... there's
diff --git a/rpython/jit/backend/ppc/codebuf.py b/rpython/jit/backend/ppc/codebuf.py
deleted file mode 100644
--- a/rpython/jit/backend/ppc/codebuf.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin
-
-class MachineCodeBlockWrapper(BlockBuilderMixin):
-    def __init__(self):
-        self.init_block_builder()
diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -1073,14 +1073,14 @@
         if IS_PPC_64:
             self.load(r.TOC.value, r.SP.value, 5 * WORD)
 
-    def make_function_prologue(self, frame_size):
-        """ Build a new stackframe of size frame_size 
-            and store the LR in the previous frame.
-        """
-        with scratch_reg(self):
-            self.store_update(r.SP.value, r.SP.value, -frame_size)
-            self.mflr(r.SCRATCH.value)
-            self.store(r.SCRATCH.value, r.SP.value, frame_size + LR_BC_OFFSET) 
+    ## def make_function_prologue(self, frame_size):
+    ##     """ Build a new stackframe of size frame_size 
+    ##         and store the LR in the previous frame.
+    ##     """
+    ##     with scratch_reg(self):
+    ##         self.store_update(r.SP.value, r.SP.value, -frame_size)
+    ##         self.mflr(r.SCRATCH.value)
+    ##         self.store(r.SCRATCH.value, r.SP.value, frame_size + LR_BC_OFFSET) 
 
     def restore_LR_from_caller_frame(self, frame_size):
         """ Restore the LR from the calling frame.
diff --git a/rpython/jit/backend/ppc/helper/assembler.py b/rpython/jit/backend/ppc/helper/assembler.py
--- a/rpython/jit/backend/ppc/helper/assembler.py
+++ b/rpython/jit/backend/ppc/helper/assembler.py
@@ -1,7 +1,6 @@
 import rpython.jit.backend.ppc.condition as c
 from rpython.rlib.rarithmetic import intmask
-from rpython.jit.backend.ppc.arch import (MAX_REG_PARAMS, IS_PPC_32, WORD,
-                                          BACKCHAIN_SIZE)
+from rpython.jit.backend.ppc.arch import MAX_REG_PARAMS, IS_PPC_32, WORD
 from rpython.jit.metainterp.history import FLOAT
 import rpython.jit.backend.ppc.register as r
 from rpython.rtyper.lltypesystem import rffi, lltype
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -5,9 +5,8 @@
 import rpython.jit.backend.ppc.register as r
 from rpython.jit.backend.ppc.locations import imm
 from rpython.jit.backend.ppc.locations import imm as make_imm_loc
-from rpython.jit.backend.ppc.arch import (IS_PPC_32, WORD, BACKCHAIN_SIZE,
-                                          MAX_REG_PARAMS, MAX_FREG_PARAMS,
-                                          FORCE_INDEX_OFS, JITFRAME_FIXED_SIZE)
+from rpython.jit.backend.ppc.arch import (IS_PPC_32, WORD,
+                                          MAX_REG_PARAMS, MAX_FREG_PARAMS)
 
 from rpython.jit.metainterp.history import (JitCellToken, TargetToken, Box,
                                             AbstractFailDescr, FLOAT, INT, REF)
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -4,13 +4,8 @@
 from rpython.jit.backend.ppc.codebuilder import (PPCBuilder, OverwritingBuilder,
                                                  scratch_reg)
 from rpython.jit.backend.ppc.arch import (IS_PPC_32, IS_PPC_64, WORD,
-                                          NONVOLATILES, MAX_REG_PARAMS,
-                                          GPR_SAVE_AREA, BACKCHAIN_SIZE,
-                                          FPR_SAVE_AREA, NONVOLATILES_FLOAT,
-                                          FLOAT_INT_CONVERSION, FORCE_INDEX,
-                                          SIZE_LOAD_IMM_PATCH_SP,
-                                          FORCE_INDEX_OFS, LR_BC_OFFSET,
-                                          JITFRAME_FIXED_SIZE)
+                                          LR_BC_OFFSET,
+                                          STD_FRAME_SIZE_IN_BYTES)
 from rpython.jit.backend.ppc.helper.assembler import Saved_Volatiles
 from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg
 import rpython.jit.backend.ppc.register as r
@@ -73,17 +68,17 @@
 
 class AssemblerPPC(OpAssembler, BaseAssembler):
 
-    ENCODING_AREA               = FORCE_INDEX_OFS
-    OFFSET_SPP_TO_GPR_SAVE_AREA = (FORCE_INDEX + FLOAT_INT_CONVERSION
-                                   + ENCODING_AREA)
-    OFFSET_SPP_TO_FPR_SAVE_AREA = (OFFSET_SPP_TO_GPR_SAVE_AREA
-                                   + GPR_SAVE_AREA)
-    OFFSET_SPP_TO_OLD_BACKCHAIN = (OFFSET_SPP_TO_GPR_SAVE_AREA
-                                   + GPR_SAVE_AREA + FPR_SAVE_AREA)
+    #ENCODING_AREA               = FORCE_INDEX_OFS
+    #OFFSET_SPP_TO_GPR_SAVE_AREA = (FORCE_INDEX + FLOAT_INT_CONVERSION
+    #                               + ENCODING_AREA)
+    #OFFSET_SPP_TO_FPR_SAVE_AREA = (OFFSET_SPP_TO_GPR_SAVE_AREA
+    #                               + GPR_SAVE_AREA)
+    #OFFSET_SPP_TO_OLD_BACKCHAIN = (OFFSET_SPP_TO_GPR_SAVE_AREA
+    #                               + GPR_SAVE_AREA + FPR_SAVE_AREA)
 
-    OFFSET_STACK_ARGS = OFFSET_SPP_TO_OLD_BACKCHAIN + BACKCHAIN_SIZE * WORD
-    if IS_PPC_64:
-        OFFSET_STACK_ARGS += MAX_REG_PARAMS * WORD
+    #OFFSET_STACK_ARGS = OFFSET_SPP_TO_OLD_BACKCHAIN + BACKCHAIN_SIZE * WORD
+    #if IS_PPC_64:
+    #    OFFSET_STACK_ARGS += MAX_REG_PARAMS * WORD
 
     def __init__(self, cpu, translate_support_code=False):
         self.cpu = cpu
@@ -676,15 +671,46 @@
             fpreg = r.MANAGED_FP_REGS[i]
             mc.stfd(fpreg.value, r.SPP.value, (i + FLOAT_OFFSET) * WORD)
 
-    def gen_bootstrap_code(self, loophead, spilling_area):
-        self._insert_stack_check()
-        self._make_frame(spilling_area)
-        self.mc.b_offset(loophead)
+    #def gen_bootstrap_code(self, loophead, spilling_area):
+    #    self._insert_stack_check()
+    #    self._make_frame(spilling_area)
+    #    self.mc.b_offset(loophead)
 
-    def _insert_stack_check(self):
+    def _call_header(self):
+        # Build a new stackframe of size STD_FRAME_SIZE_IN_BYTES
+        self.mc.store_update(r.SP.value, r.SP.value, -STD_FRAME_SIZE_IN_BYTES)
+        self.mc.mflr(r.SCRATCH.value)
+        self.mc.store(r.SCRATCH.value, r.SP.value,
+                      STD_FRAME_SIZE_IN_BYTES + LR_BC_OFFSET)
+
+        XXXX
+        # save SPP at the bottom of the stack frame
+        self.mc.store(r.SPP.value, r.SP.value, WORD)
+
+        # compute spilling pointer (SPP)
+        self.mc.addi(r.SPP.value, r.SP.value, 
+                frame_depth - self.OFFSET_SPP_TO_OLD_BACKCHAIN)
+
+        # save nonvolatile registers
+        self._save_nonvolatiles()
+
+        # save r31, use r30 as scratch register
+        # this is safe because r30 has been saved already
+        assert NONVOLATILES[-1] == r.SPP
+        ofs_to_r31 = (self.OFFSET_SPP_TO_GPR_SAVE_AREA +
+                      WORD * (len(NONVOLATILES)-1))
+        self.mc.load(r.r30.value, r.SP.value, WORD)
+        self.mc.store(r.r30.value, r.SPP.value, ofs_to_r31)
+        gcrootmap = self.cpu.gc_ll_descr.gcrootmap
+        if gcrootmap and gcrootmap.is_shadow_stack:
+            self.gen_shadowstack_header(gcrootmap)
+
+    def _call_header_with_stack_check(self):
+        self._call_header()
         if self.stack_check_slowpath == 0:
             pass            # not translated
         else:
+            XXXX
             # this is the size for the miniframe
             frame_size = BACKCHAIN_SIZE * WORD
 
@@ -733,24 +759,23 @@
             pmc.bc(4, 1, offset) # jump if SCRATCH <= r16, i. e. not(SCRATCH > r16)
             pmc.overwrite()
 
-    def setup(self, looptoken, operations):
-        self.current_clt = looptoken.compiled_loop_token 
-        operations = self.cpu.gc_ll_descr.rewrite_assembler(self.cpu, 
-                operations, self.current_clt.allgcrefs)
-        assert self.memcpy_addr != 0
+    def setup(self, looptoken):
+        BaseAssembler.setup(self, looptoken)
+        assert self.memcpy_addr != 0, "setup_once() not called?"
+        self.current_clt = looptoken.compiled_loop_token
+        self.pending_guard_tokens = []
+        #if WORD == 8:
+        #    self.pending_memoryerror_trampoline_from = []
+        #    self.error_trampoline_64 = 0
         self.mc = PPCBuilder()
-        self.pending_guards = []
+        #assert self.datablockwrapper is None --- but obscure case
+        # possible, e.g. getting MemoryError and continuing
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
                                                         allblocks)
-        self.max_stack_params = 0
         self.target_tokens_currently_compiling = {}
         self.frame_depth_to_patch = []
-        self._finish_gcmap = lltype.nullptr(jitframe.GCMAP)
-        return operations # do we really need this return?
-
-    def setup_once(self):
-        BaseAssembler.setup_once(self)
+        #self.max_stack_params = 0
 
     def _append_debugging_code(self, operations, tp, number, token):
         counter = self._register_counter(tp, number, token)
@@ -788,77 +813,74 @@
         baseofs = self.cpu.get_baseofs_of_frame_field()
         self.current_clt.frame_info.update_frame_depth(baseofs, frame_depth)
 
-    def assemble_loop(self, loopname, inputargs, operations, looptoken, log):
+    @rgc.no_release_gil
+    def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs,
+                      operations, looptoken, log):
         clt = CompiledLoopToken(self.cpu, looptoken.number)
-        clt.allgcrefs = []
         looptoken.compiled_loop_token = clt
         clt._debug_nbargs = len(inputargs)
-
         if not we_are_translated():
+            # Arguments should be unique
             assert len(set(inputargs)) == len(inputargs)
 
-        operations = self.setup(looptoken, operations)
+        self.setup(looptoken)
+        frame_info = self.datablockwrapper.malloc_aligned(
+            jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
+        clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info)
+        clt.allgcrefs = []
+        clt.frame_info.clear() # for now
 
         if log:
             operations = self._inject_debugging_code(looptoken, operations,
                                                      'e', looptoken.number)
 
-        self.startpos = self.mc.currpos()
         regalloc = Regalloc(assembler=self)
+        #
+        self._call_header_with_stack_check()
+        operations = regalloc.prepare_loop(inputargs, operations,
+                                           looptoken, clt.allgcrefs)
+        looppos = self.mc.get_relative_pos()
+        frame_depth_no_fixed_size = self._assemble(regalloc, inputargs,
+                                                   operations)
+        self.update_frame_depth(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
+        #
+        size_excluding_failure_stuff = self.mc.get_relative_pos()
+        self.write_pending_failure_recoveries()
+        full_size = self.mc.get_relative_pos()
+        #
+        rawstart = self.materialize_loop(looptoken)
+        self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
+                                rawstart)
+        looptoken._ll_loop_code = looppos + rawstart
+        debug_start("jit-backend-addr")
+        debug_print("Loop %d (%s) has address 0x%x to 0x%x (bootstrap 0x%x)" % (
+            looptoken.number, loopname,
+            r_uint(rawstart + looppos),
+            r_uint(rawstart + size_excluding_failure_stuff),
+            r_uint(rawstart)))
+        debug_stop("jit-backend-addr")
+        self.patch_pending_failure_recoveries(rawstart)
+        #
+        ops_offset = self.mc.ops_offset
+        if not we_are_translated():
+            # used only by looptoken.dump() -- useful in tests
+            looptoken._x86_rawstart = rawstart
+            looptoken._x86_fullsize = full_size
+            looptoken._x86_ops_offset = ops_offset
+        looptoken._ll_function_addr = rawstart
+        if logger:
+            logger.log_loop(inputargs, operations, 0, "rewritten",
+                            name=loopname, ops_offset=ops_offset)
 
-        regalloc.prepare_loop(inputargs, operations, looptoken)
-
-        start_pos = self.mc.currpos()
-        looptoken._ppc_loop_code = start_pos
-        clt.frame_depth = clt.param_depth = -1
-        spilling_area, param_depth = self._assemble(operations, regalloc)
-        size_excluding_failure_stuff = self.mc.currpos()
-        clt.frame_depth = spilling_area
-        clt.param_depth = param_depth
-        frame_info = self.datablockwrapper.malloc_aligned(jitframe.JITFRAMEINFO_SIZE,
-                                                          alignment=WORD)
-        clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info)
-
-        direct_bootstrap_code = self.mc.currpos()
-        frame_depth = self.compute_frame_depth(spilling_area, param_depth)
-        frame_depth += JITFRAME_FIXED_SIZE
-        self.update_frame_depth(frame_depth)
-        self.gen_bootstrap_code(start_pos, frame_depth)
-
-        self.write_pending_failure_recoveries()
-        if IS_PPC_64:
-            fdescr = self.gen_64_bit_func_descr()
-
-        # write instructions to memory
-        loop_start = self.materialize_loop(looptoken, False)
-        self.fixup_target_tokens(loop_start)
-
-        real_start = loop_start + direct_bootstrap_code
-        if IS_PPC_32:
-            looptoken._ppc_func_addr = looptoken._ll_function_addr = real_start
-        else:
-            self.write_64_bit_func_descr(fdescr, real_start)
-            looptoken._ppc_func_addr = looptoken._ll_function_addr = fdescr
-
-        self.process_pending_guards(loop_start)
-
-        if log and not we_are_translated():
-            self.mc._dump_trace(real_start,
-                    'loop.asm')
-
-        ops_offset = self.mc.ops_offset
+        self.fixup_target_tokens(rawstart)
         self.teardown()
-
-        debug_start("jit-backend-addr")
-        debug_print("Loop %d (%s) has address %x to %x (bootstrap %x)" % (
-            looptoken.number, loopname,
-            real_start,
-            real_start + size_excluding_failure_stuff,
-            loop_start))
-        debug_stop("jit-backend-addr")
-
-        return AsmInfo(ops_offset, loop_start, 
-                size_excluding_failure_stuff - start_pos)
+        # oprofile support
+        if self.cpu.profile_agent is not None:
+            name = "Loop # %s: %s" % (looptoken.number, loopname)
+            self.cpu.profile_agent.native_code_written(name,
+                                                       rawstart, full_size)
+        return AsmInfo(ops_offset, rawstart + looppos,
+                       size_excluding_failure_stuff - looppos)
 
     def _assemble(self, operations, regalloc):
         regalloc.compute_hint_frame_locations(operations)
diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py
--- a/rpython/jit/backend/ppc/register.py
+++ b/rpython/jit/backend/ppc/register.py
@@ -17,8 +17,10 @@
 VOLATILES           = [r0, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12]
 # volatile r2 is persisted around calls and r13 can be ignored
 
-NONVOLATILES_FLOAT  = [f14, f15, f16, f17, f18, f19, f20, f21, f22, f23,
-                    f24, f25, f26, f27, f28, f29, f30, f31]
+# we don't use any non-volatile float register, to keep the frame header
+# code short-ish
+#NONVOLATILES_FLOAT  = [f14, f15, f16, f17, f18, f19, f20, f21, f22, f23,
+#                    f24, f25, f26, f27, f28, f29, f30, f31]
 VOLATILES_FLOAT  = [f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13]
 
 SCRATCH    = r0
@@ -28,12 +30,12 @@
 RES        = r3
 SPP        = r31
 
-MANAGED_REGS = [r3, r4, r5, r6, r7, r8, r9, r10,
-                r11, r12, r14, r15, r16, r17, r18, 
-                r19, r20, r21, r22, r23, r24, r25, r26,
-                r27, r28, r29, r30]
+MANAGED_REGS = [r3, r4, r5, r6, r7, r8, r9, r10, r11, r12,
+                r25, r26, r27, r28, r29, r30]
+                # registers r14 to r24 are not touched, we have enough
+                # registers already
 
-MANAGED_FP_REGS = VOLATILES_FLOAT[1:] + NONVOLATILES_FLOAT
+MANAGED_FP_REGS = VOLATILES_FLOAT[1:] #+ NONVOLATILES_FLOAT
 
 PARAM_REGS = [r3, r4, r5, r6, r7, r8, r9, r10]
 PARAM_FPREGS = [f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13]
diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -1,7 +1,7 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
 from rpython.rtyper.llinterp import LLInterpreter
-from rpython.jit.backend.ppc.arch import FORCE_INDEX_OFS
+#from rpython.jit.backend.ppc.arch import FORCE_INDEX_OFS
 from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU
 from rpython.jit.backend.ppc.ppc_assembler import AssemblerPPC
 from rpython.jit.backend.ppc.arch import WORD
@@ -43,10 +43,6 @@
     def finish_once(self):
         self.assembler.finish_once()
 
-    def compile_loop(self, inputargs, operations, looptoken, log=True, name=""):
-        return self.assembler.assemble_loop(name, inputargs,
-                                      operations, looptoken, log)
-
     def compile_bridge(self, faildescr, inputargs, operations,
                       original_loop_token, log=False):
         clt = original_loop_token.compiled_loop_token
@@ -54,12 +50,6 @@
         return self.assembler.assemble_bridge(faildescr, inputargs, operations,
                                        original_loop_token, log=log)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     @staticmethod
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)

From noreply at buildbot.pypy.org  Fri Aug 21 10:50:45 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 10:50:45 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-updated-backend: copy copy copy from x86
Message-ID: <20150821085045.D15DD1C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: ppc-updated-backend
Changeset: r79094:e7a2b3e7bbff
Date: 2015-08-20 09:14 -0700
http://bitbucket.org/pypy/pypy/changeset/e7a2b3e7bbff/

Log:	copy copy copy from x86

diff --git a/rpython/jit/backend/ppc/arch.py b/rpython/jit/backend/ppc/arch.py
--- a/rpython/jit/backend/ppc/arch.py
+++ b/rpython/jit/backend/ppc/arch.py
@@ -61,15 +61,21 @@
 # register r31.
 
 
+LR_BC_OFFSET            = 16
+PARAM_SAVE_AREA_OFFSET  = 48
+THREADLOCAL_ADDR_OFFSET = 112
+GPR_SAVE_AREA_OFFSET    = 120
+
 REGISTERS_SAVED         = [r.r25, r.r26, r.r27, r.r28, r.r29, r.r30, r.r31]
 assert REGISTERS_SAVED == [_r for _r in r.NONVOLATILES
                               if _r in r.MANAGED_REGS or _r == r.r31]
 
-STD_FRAME_SIZE_IN_BYTES = 120 + len(REGISTERS_SAVED) * WORD
+STD_FRAME_SIZE_IN_BYTES = GPR_SAVE_AREA_OFFSET + len(REGISTERS_SAVED) * WORD
 assert STD_FRAME_SIZE_IN_BYTES % 16 == 0
 
-# offset to LR in BACKCHAIN
-if IS_PPC_32:
-    LR_BC_OFFSET = WORD
-else:
-    LR_BC_OFFSET = 16
+
+# The JITFRAME_FIXED_SIZE is measured in words, and should be the
+# number of registers that need to be saved into the jitframe when
+# failing a guard, for example.  (Note: it is about the jitframe,
+# not the frame.)
+JITFRAME_FIXED_SIZE = len(r.MANAGED_REGS) + len(r.MANAGED_FP_REGS)
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -4,8 +4,11 @@
 from rpython.jit.backend.ppc.codebuilder import (PPCBuilder, OverwritingBuilder,
                                                  scratch_reg)
 from rpython.jit.backend.ppc.arch import (IS_PPC_32, IS_PPC_64, WORD,
-                                          LR_BC_OFFSET,
-                                          STD_FRAME_SIZE_IN_BYTES)
+                                          LR_BC_OFFSET, REGISTERS_SAVED,
+                                          GPR_SAVE_AREA_OFFSET,
+                                          THREADLOCAL_ADDR_OFFSET,
+                                          STD_FRAME_SIZE_IN_BYTES,
+                                          JITFRAME_FIXED_SIZE)
 from rpython.jit.backend.ppc.helper.assembler import Saved_Volatiles
 from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg
 import rpython.jit.backend.ppc.register as r
@@ -30,6 +33,7 @@
 from rpython.jit.backend.ppc.locations import StackLocation, get_spp_offset, imm
 from rpython.rlib.jit import AsmInfo
 from rpython.rlib.objectmodel import compute_unique_id
+from rpython.rlib.rarithmetic import r_uint
 
 memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address,
                                        rffi.SIZE_T], lltype.Void,
@@ -128,6 +132,7 @@
     # The code generated here allocates a new stackframe 
     # and is the first machine code to be executed.
     def _make_frame(self, frame_depth):
+        XXX
         self.mc.make_function_prologue(frame_depth)
 
         # save SPP at the bottom of the stack frame
@@ -683,26 +688,20 @@
         self.mc.store(r.SCRATCH.value, r.SP.value,
                       STD_FRAME_SIZE_IN_BYTES + LR_BC_OFFSET)
 
-        XXXX
-        # save SPP at the bottom of the stack frame
-        self.mc.store(r.SPP.value, r.SP.value, WORD)
+        # save registers r25 to r31
+        for i, reg in enumerate(REGISTERS_SAVED):
+            self.mc.store(reg.value, r.SP.value,
+                          GPR_SAVE_AREA_OFFSET + i * WORD)
 
-        # compute spilling pointer (SPP)
-        self.mc.addi(r.SPP.value, r.SP.value, 
-                frame_depth - self.OFFSET_SPP_TO_OLD_BACKCHAIN)
+        # save r4, the second argument, to THREADLOCAL_ADDR_OFFSET
+        self.mc.store(r.r4.value, r.SP.value, THREADLOCAL_ADDR_OFFSET)
 
-        # save nonvolatile registers
-        self._save_nonvolatiles()
+        # move r3, the first argument, to r31 (SPP): the jitframe object
+        self.mc.mr(r.SPP.value, r.r3.value)
 
-        # save r31, use r30 as scratch register
-        # this is safe because r30 has been saved already
-        assert NONVOLATILES[-1] == r.SPP
-        ofs_to_r31 = (self.OFFSET_SPP_TO_GPR_SAVE_AREA +
-                      WORD * (len(NONVOLATILES)-1))
-        self.mc.load(r.r30.value, r.SP.value, WORD)
-        self.mc.store(r.r30.value, r.SPP.value, ofs_to_r31)
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
+            XXX
             self.gen_shadowstack_header(gcrootmap)
 
     def _call_header_with_stack_check(self):
@@ -813,6 +812,10 @@
         baseofs = self.cpu.get_baseofs_of_frame_field()
         self.current_clt.frame_info.update_frame_depth(baseofs, frame_depth)
 
+    def patch_stack_checks(self, framedepth, rawstart):
+        for ofs in self.frame_depth_to_patch:
+            self._patch_frame_depth(ofs + rawstart, framedepth)
+
     @rgc.no_release_gil
     def assemble_loop(self, jd_id, unique_id, logger, loopname, inputargs,
                       operations, looptoken, log):
@@ -864,9 +867,9 @@
         ops_offset = self.mc.ops_offset
         if not we_are_translated():
             # used only by looptoken.dump() -- useful in tests
-            looptoken._x86_rawstart = rawstart
-            looptoken._x86_fullsize = full_size
-            looptoken._x86_ops_offset = ops_offset
+            looptoken._ppc_rawstart = rawstart
+            looptoken._ppc_fullsize = full_size
+            looptoken._ppc_ops_offset = ops_offset
         looptoken._ll_function_addr = rawstart
         if logger:
             logger.log_loop(inputargs, operations, 0, "rewritten",
@@ -875,26 +878,26 @@
         self.fixup_target_tokens(rawstart)
         self.teardown()
         # oprofile support
-        if self.cpu.profile_agent is not None:
-            name = "Loop # %s: %s" % (looptoken.number, loopname)
-            self.cpu.profile_agent.native_code_written(name,
-                                                       rawstart, full_size)
+        #if self.cpu.profile_agent is not None:
+        #    name = "Loop # %s: %s" % (looptoken.number, loopname)
+        #    self.cpu.profile_agent.native_code_written(name,
+        #                                               rawstart, full_size)
         return AsmInfo(ops_offset, rawstart + looppos,
                        size_excluding_failure_stuff - looppos)
 
-    def _assemble(self, operations, regalloc):
+    def _assemble(self, regalloc, inputargs, operations):
+        self._regalloc = regalloc
         regalloc.compute_hint_frame_locations(operations)
-        self._walk_operations(operations, regalloc)
-        frame_depth = regalloc.frame_manager.get_frame_depth()
-        param_depth = self.max_stack_params
+        regalloc.walk_operations(inputargs, operations)
+        if 1: # we_are_translated() or self.cpu.dont_keepalive_stuff:
+            self._regalloc = None   # else keep it around for debugging
+        frame_depth = regalloc.get_final_frame_depth()
         jump_target_descr = regalloc.jump_target_descr
         if jump_target_descr is not None:
-            frame_depth = max(frame_depth,
-                              jump_target_descr._ppc_clt.frame_depth)
-            param_depth = max(param_depth, 
-                              jump_target_descr._ppc_clt.param_depth)
-        return frame_depth, param_depth
-
+            tgt_depth = jump_target_descr._ppc_clt.frame_info.jfi_frame_depth
+            target_frame_depth = tgt_depth - JITFRAME_FIXED_SIZE
+            frame_depth = max(frame_depth, target_frame_depth)
+        return frame_depth
 
     def assemble_bridge(self, faildescr, inputargs, operations, looptoken, log):
         if not we_are_translated():
@@ -1032,64 +1035,6 @@
         assert self.datablockwrapper is None
         self.max_stack_params = 0
 
-    def _walk_operations(self, operations, regalloc):
-        self._regalloc = regalloc
-        while regalloc.position() < len(operations) - 1:
-            regalloc.next_instruction()
-            pos = regalloc.position()
-            op = operations[pos]
-            opnum = op.getopnum()
-            if op.has_no_side_effect() and op.result not in regalloc.longevity:
-                regalloc.possibly_free_vars_for_op(op)
-            elif self.can_merge_with_next_guard(op, pos, operations)\
-                    and opnum in (rop.CALL_RELEASE_GIL, rop.CALL_ASSEMBLER,\
-                    rop.CALL_MAY_FORCE):  # XXX fix  
-                guard = operations[pos + 1]
-                assert guard.is_guard()
-                arglocs = regalloc.operations_with_guard[opnum](regalloc, op,
-                                                                guard)
-                operations_with_guard[opnum](self, op,
-                                             guard, arglocs, regalloc)
-                regalloc.next_instruction()
-                regalloc.possibly_free_vars_for_op(guard)
-                regalloc.possibly_free_vars(guard.getfailargs())
-            elif not we_are_translated() and op.getopnum() == -124:
-                regalloc.prepare_force_spill(op)
-            else:
-                arglocs = regalloc.operations[opnum](regalloc, op)
-                if arglocs is not None:
-                    self.operations[opnum](self, op, arglocs, regalloc)
-            if op.is_guard():
-                regalloc.possibly_free_vars(op.getfailargs())
-            if op.result:
-                regalloc.possibly_free_var(op.result)
-            regalloc.possibly_free_vars_for_op(op)
-            regalloc.free_temp_vars()
-            regalloc._check_invariants()
-
-    def can_merge_with_next_guard(self, op, i, operations):
-        if (op.getopnum() == rop.CALL_MAY_FORCE or
-            op.getopnum() == rop.CALL_ASSEMBLER or
-            op.getopnum() == rop.CALL_RELEASE_GIL):
-            assert operations[i + 1].getopnum() == rop.GUARD_NOT_FORCED
-            return True
-        if not op.is_comparison():
-            if op.is_ovf():
-                if (operations[i + 1].getopnum() != rop.GUARD_NO_OVERFLOW and
-                    operations[i + 1].getopnum() != rop.GUARD_OVERFLOW):
-                    assert 0, "int_xxx_ovf not followed by guard_(no)_overflow"
-                return True
-            return False
-        if (operations[i + 1].getopnum() != rop.GUARD_TRUE and
-            operations[i + 1].getopnum() != rop.GUARD_FALSE):
-            return False
-        if operations[i + 1].getarg(0) is not op.result:
-            return False
-        if (self._regalloc.longevity[op.result][1] > i + 1 or
-            op.result in operations[i + 1].getfailargs()):
-            return False
-        return True
-
     def gen_64_bit_func_descr(self):
         return self.datablockwrapper.malloc_aligned(3*WORD, alignment=1)
 
@@ -1138,7 +1083,6 @@
         allblocks = self.get_asmmemmgr_blocks(looptoken)
         start = self.mc.materialize(self.cpu, allblocks,
                                     self.cpu.gc_ll_descr.gcrootmap)
-        #from pypy.rlib.rarithmetic import r_uint
         #print "=== Loop start is at %s ===" % hex(r_uint(start))
         return start
 
@@ -1160,9 +1104,16 @@
         return startpos
 
     def write_pending_failure_recoveries(self):
-        for tok in self.pending_guards:
+        # for each pending guard, generate the code of the recovery stub
+        # at the end of self.mc.
+        for tok in self.pending_guard_tokens:
             tok.pos_recovery_stub = self.generate_quick_failure(tok)
 
+    def patch_pending_failure_recoveries(self, rawstart):
+        clt = self.current_clt
+        for tok in self.pending_guard_tokens:
+            xxxxxxxxx
+
     def process_pending_guards(self, block_start):
         clt = self.current_clt
         for tok in self.pending_guards:
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -198,26 +198,35 @@
 
     def __init__(self, assembler=None):
         self.cpu = assembler.cpu
-        self.frame_manager = PPCFrameManager(self.cpu.get_baseofs_of_frame_field())
+        #self.frame_manager = PPCFrameManager(self.cpu.get_baseofs_of_frame_field())
         self.assembler = assembler
         self.jump_target_descr = None
         self.final_jump_op = None
 
-    def _prepare(self,  inputargs, operations):
-        self.fm = self.frame_manager
-        longevity, last_real_usage = compute_vars_longevity(inputargs,
-                                                            operations)
+    def _prepare(self,  inputargs, operations, allgcrefs):
+        cpu = self.assembler.cpu
+        self.fm = PPCFrameManager(cpu.get_baseofs_of_frame_field())
+        operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations,
+                                                       allgcrefs)
+        # compute longevity of variables
+        longevity, last_real_usage = compute_vars_longevity(
+                                                    inputargs, operations)
         self.longevity = longevity
         self.last_real_usage = last_real_usage
-        fm = self.frame_manager
-        asm = self.assembler
-        self.fprm = FPRegisterManager(longevity, fm, asm)
-        self.rm = PPCRegisterManager(longevity, fm, asm)
+        self.rm = PPCRegisterManager(self.longevity,
+                                     frame_manager = self.fm,
+                                     assembler = self.assembler)
+        self.fprm = FPRegisterManager(self.longevity, frame_manager = self.fm,
+                                      assembler = self.assembler)
+        return operations
 
-    def prepare_loop(self, inputargs, operations, looptoken):
-        self._prepare(inputargs, operations)
+    def prepare_loop(self, inputargs, operations, looptoken, allgcrefs):
+        operations = self._prepare(inputargs, operations, allgcrefs)
         self._set_initial_bindings(inputargs, looptoken)
-        self.possibly_free_vars(inputargs)
+        # note: we need to make a copy of inputargs because possibly_free_vars
+        # is also used on op args, which is a non-resizable list
+        self.possibly_free_vars(list(inputargs))
+        return operations
 
     def prepare_bridge(self, inputargs, arglocs, ops):
         self._prepare(inputargs, ops)
@@ -251,6 +260,9 @@
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
 
+    def get_final_frame_depth(self):
+        return self.fm.get_frame_depth()
+
     def possibly_free_var(self, var):
         if var.type == FLOAT:
             self.fprm.possibly_free_var(var)
@@ -289,9 +301,35 @@
                         forbidden_vars=forbidden_vars,
                         selected_reg=selected_reg)
 
-    def _check_invariants(self):
-        self.rm._check_invariants()
-        self.fprm._check_invariants()
+    def walk_operations(self, inputargs, operations):
+        i = 0
+        #self.operations = operations
+        while i < len(operations):
+            op = operations[i]
+            self.assembler.mc.mark_op(op)
+            self.rm.position = i
+            self.fprm.position = i
+            if op.has_no_side_effect() and op.result not in self.longevity:
+                i += 1
+                self.possibly_free_vars_for_op(op)
+                continue
+            if self.can_merge_with_next_guard(op, i, operations):
+                oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
+                i += 1
+            elif not we_are_translated() and op.getopnum() == -124:
+                self._consider_force_spill(op)
+            else:
+                oplist[op.getopnum()](self, op)
+            self.possibly_free_vars_for_op(op)
+            self.rm._check_invariants()
+            self.fprm._check_invariants()
+            i += 1
+        assert not self.rm.reg_bindings
+        assert not self.fprm.reg_bindings
+        #self.flush_loop()
+        self.assembler.mc.mark_op(None) # end of the loop
+        for arg in inputargs:
+            self.possibly_free_var(arg)
 
     def loc(self, var):
         if var.type == FLOAT:
@@ -299,9 +337,6 @@
         else:
             return self.rm.loc(var)
 
-    def position(self):
-        return self.rm.position
-
     def next_instruction(self):
         self.rm.next_instruction()
         self.fprm.next_instruction()
@@ -681,11 +716,10 @@
         return []
 
     def prepare_setfield_gc(self, op):
-        boxes = op.getarglist()
-        a0, a1 = boxes
         ofs, size, sign = unpack_fielddescr(op.getdescr())
-        base_loc = self._ensure_value_is_boxed(a0, boxes)
-        value_loc = self._ensure_value_is_boxed(a1, boxes)
+        args = op.getarglist()
+        base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args)
+        value_loc = self.make_sure_var_in_reg(op.getarg(1), args)
         if _check_imm_arg(ofs):
             ofs_loc = imm(ofs)
         else:
@@ -1103,8 +1137,8 @@
 
 
 
-operations = [notimplemented] * (rop._LAST + 1)
-operations_with_guard = [notimplemented_with_guard] * (rop._LAST + 1)
+oplist = [notimplemented] * (rop._LAST + 1)
+oplist_with_guard = [notimplemented_with_guard] * (rop._LAST + 1)
 
 def get_scale(size):
     scale = 0
@@ -1120,7 +1154,7 @@
     methname = 'prepare_%s' % key
     if hasattr(Regalloc, methname):
         func = getattr(Regalloc, methname).im_func
-        operations[value] = func
+        oplist[value] = func
 
 for key, value in rop.__dict__.items():
     key = key.lower()
@@ -1129,8 +1163,5 @@
     methname = 'prepare_guard_%s' % key
     if hasattr(Regalloc, methname):
         func = getattr(Regalloc, methname).im_func
-        operations_with_guard[value] = func
-        operations[value] = add_none_argument(func)
-
-Regalloc.operations = operations
-Regalloc.operations_with_guard = operations_with_guard
+        oplist_with_guard[value] = func
+        oplist[value] = add_none_argument(func)

From noreply at buildbot.pypy.org  Fri Aug 21 10:50:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 10:50:47 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-updated-backend: One test passes!
Message-ID: <20150821085047.DB9881C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: ppc-updated-backend
Changeset: r79095:4865720d70df
Date: 2015-08-21 01:03 -0700
http://bitbucket.org/pypy/pypy/changeset/4865720d70df/

Log:	One test passes!

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -282,6 +282,7 @@
                     ll_threadlocal_addr = rffi.cast(llmemory.Address,
                         self._debug_errno_container)
                 llop.gc_writebarrier(lltype.Void, ll_frame)
+                import pdb;pdb.set_trace()
                 ll_frame = func(ll_frame, ll_threadlocal_addr)
             finally:
                 if not self.translate_support_code:
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -346,10 +346,46 @@
 
     _mixin_ = True
 
+    def emit_increment_debug_counter(self, op, arglocs, regalloc):
+        [addr_loc, value_loc] = arglocs
+        self.mc.load(value_loc.value, addr_loc.value, 0)
+        self.mc.addi(value_loc.value, value_loc.value, 1)
+        self.mc.store(value_loc.value, addr_loc.value, 0)
+
     def emit_finish(self, op, arglocs, regalloc):
-        [argloc] = arglocs
-        self._gen_epilogue(self.mc)
-        # ARM has a fcond argument to this function, returned at the end
+        base_ofs = self.cpu.get_baseofs_of_frame_field()
+        if len(arglocs) == 2:
+            [return_val, fail_descr_loc] = arglocs
+            self.mc.std(return_val.value, r.SPP.value, base_ofs)
+        else:
+            [fail_descr_loc] = arglocs
+
+        ofs = self.cpu.get_ofs_of_frame_field('jf_descr')
+        self.mc.load_imm(r.r5, fail_descr_loc.getint())
+        self.mc.std(r.r5.value, r.SPP.value, ofs)
+
+        ## arglist = op.getarglist()
+        ## if arglist and arglist[0].type == REF:
+        ##     if self._finish_gcmap:
+        ##         # we're returning with a guard_not_forced_2, and
+        ##         # additionally we need to say that eax/rax contains
+        ##         # a reference too:
+        ##         self._finish_gcmap[0] |= r_uint(1)
+        ##         gcmap = self._finish_gcmap
+        ##     else:
+        ##         gcmap = self.gcmap_for_finish
+        ##     self.push_gcmap(self.mc, gcmap, store=True)
+        ## elif self._finish_gcmap:
+        ##     # we're returning with a guard_not_forced_2
+        ##     gcmap = self._finish_gcmap
+        ##     self.push_gcmap(self.mc, gcmap, store=True)
+        ## else:
+        ##     # note that the 0 here is redundant, but I would rather
+        ##     # keep that one and kill all the others
+        ##     ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
+        ##     self.mc.MOV_bi(ofs, 0)
+        # exit function
+        self._call_footer()
 
     def emit_jump(self, op, arglocs, regalloc):
         # The backend's logic assumes that the target code is in a piece of
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -85,23 +85,11 @@
     #    OFFSET_STACK_ARGS += MAX_REG_PARAMS * WORD
 
     def __init__(self, cpu, translate_support_code=False):
-        self.cpu = cpu
-        self.mc = None
-        self.memcpy_addr = 0
-        self.pending_guards = None
-        self.fail_boxes_count = 0
-        self.current_clt = None
-        self.malloc_slowpath = 0
-        self.wb_slowpath = [0, 0, 0, 0]
-        self._regalloc = None
-        self.datablockwrapper = None
-        self.max_stack_params = 0
-        self.propagate_exception_path = 0
+        BaseAssembler.__init__(self, cpu, translate_support_code)
+        self.loop_run_counters = []
+        self.setup_failure_recovery()
         self.stack_check_slowpath = 0
-        self.setup_failure_recovery()
-        self._debug = False
-        self.loop_run_counters = []
-        self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i')
+        self.teardown()
 
     def set_debug(self, v):
         self._debug = v
@@ -645,11 +633,12 @@
         return mc.materialize(self.cpu, [], self.cpu.gc_ll_descr.gcrootmap)
 
     def _gen_epilogue(self, mc):
+        XXX
         gcrootmap = self.cpu.gc_ll_descr.gcrootmap
         if gcrootmap and gcrootmap.is_shadow_stack:
             self.gen_footer_shadowstack(gcrootmap, mc)
-        # save SPP in r5
-        # (assume that r5 has been written to failboxes)
+
+        # save SPP back in r3
         mc.mr(r.r5.value, r.SPP.value)
         self._restore_nonvolatiles(mc, r.r5)
         # load old backchain into r4
@@ -682,6 +671,12 @@
     #    self.mc.b_offset(loophead)
 
     def _call_header(self):
+        if IS_PPC_64:
+            # Reserve space for a function descriptor, 3 words
+            self.mc.write64(0)
+            self.mc.write64(0)
+            self.mc.write64(0)
+
         # Build a new stackframe of size STD_FRAME_SIZE_IN_BYTES
         self.mc.store_update(r.SP.value, r.SP.value, -STD_FRAME_SIZE_IN_BYTES)
         self.mc.mflr(r.SCRATCH.value)
@@ -758,6 +753,27 @@
             pmc.bc(4, 1, offset) # jump if SCRATCH <= r16, i. e. not(SCRATCH > r16)
             pmc.overwrite()
 
+    def _call_footer(self):
+        # the return value is the jitframe
+        self.mc.mr(r.r3.value, r.SPP.value)
+
+        gcrootmap = self.cpu.gc_ll_descr.gcrootmap
+        if gcrootmap and gcrootmap.is_shadow_stack:
+            self._call_footer_shadowstack(gcrootmap)
+
+        # load old backchain into r4
+        self.mc.load(r.r4.value, r.SP.value,
+                     STD_FRAME_SIZE_IN_BYTES + LR_BC_OFFSET)
+
+        # restore registers r25 to r31
+        for i, reg in enumerate(REGISTERS_SAVED):
+            self.mc.load(reg.value, r.SP.value,
+                         GPR_SAVE_AREA_OFFSET + i * WORD)
+
+        self.mc.addi(r.SP.value, r.SP.value, STD_FRAME_SIZE_IN_BYTES)
+        self.mc.mtlr(r.r4.value)     # restore LR
+        self.mc.blr()
+
     def setup(self, looptoken):
         BaseAssembler.setup(self, looptoken)
         assert self.memcpy_addr != 0, "setup_once() not called?"
@@ -776,38 +792,6 @@
         self.frame_depth_to_patch = []
         #self.max_stack_params = 0
 
-    def _append_debugging_code(self, operations, tp, number, token):
-        counter = self._register_counter(tp, number, token)
-        c_adr = ConstInt(rffi.cast(lltype.Signed, counter))
-        box = BoxInt()
-        box2 = BoxInt()
-        ops = [ResOperation(rop.GETFIELD_RAW, [c_adr],
-                            box, descr=self.debug_counter_descr),
-               ResOperation(rop.INT_ADD, [box, ConstInt(1)], box2),
-               ResOperation(rop.SETFIELD_RAW, [c_adr, box2],
-                            None, descr=self.debug_counter_descr)]
-        operations.extend(ops)
-
-    @specialize.argtype(1)
-    def _inject_debugging_code(self, looptoken, operations, tp, number):
-        if self._debug:
-            # before doing anything, let's increase a counter
-            s = 0
-            for op in operations:
-                s += op.getopnum()
-            looptoken._ppc_debug_checksum = s
-
-            newoperations = []
-            self._append_debugging_code(newoperations, tp, number,
-                                        None)
-            for op in operations:
-                newoperations.append(op)
-                if op.getopnum() == rop.LABEL:
-                    self._append_debugging_code(newoperations, 'l', number,
-                                                op.getdescr())
-            operations = newoperations
-        return operations
-
     def update_frame_depth(self, frame_depth):
         baseofs = self.cpu.get_baseofs_of_frame_field()
         self.current_clt.frame_info.update_frame_depth(baseofs, frame_depth)
@@ -852,6 +836,9 @@
         full_size = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(looptoken)
+        if IS_PPC_64:    # fix the function descriptor (3 words)
+            rffi.cast(rffi.LONGP, rawstart)[0] = rawstart + 3 * WORD
+        #
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         looptoken._ll_loop_code = looppos + rawstart
@@ -1027,22 +1014,9 @@
         return size
 
     def teardown(self):
-        self.patch_list = None
-        self.pending_guards = None
+        self.pending_guard_tokens = None
+        self.mc = None
         self.current_clt = None
-        self.mc = None
-        self._regalloc = None
-        assert self.datablockwrapper is None
-        self.max_stack_params = 0
-
-    def gen_64_bit_func_descr(self):
-        return self.datablockwrapper.malloc_aligned(3*WORD, alignment=1)
-
-    def write_64_bit_func_descr(self, descr, start_addr):
-        data = rffi.cast(rffi.CArrayPtr(lltype.Signed), descr)
-        data[0] = start_addr
-        data[1] = 0
-        data[2] = 0
 
     def compute_frame_depth(self, spilling_area, param_depth):
         PARAMETER_AREA = param_depth * WORD
@@ -1433,8 +1407,5 @@
         func = getattr(AssemblerPPC, methname).im_func
         operations_with_guard[value] = func
 
-AssemblerPPC.operations = operations
-AssemblerPPC.operations_with_guard = operations_with_guard
-
 class BridgeAlreadyCompiled(Exception):
     pass
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -30,6 +30,7 @@
 from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.rlib import rgc
 
 # xxx hack: set a default value for TargetToken._arm_loop_code.  If 0, we know
 # that it is a LABEL that was not compiled yet.
@@ -302,6 +303,9 @@
                         selected_reg=selected_reg)
 
     def walk_operations(self, inputargs, operations):
+        from rpython.jit.backend.ppc.ppc_assembler import (
+            operations_with_guard as asm_operations_with_guard,
+            operations as asm_operations)
         i = 0
         #self.operations = operations
         while i < len(operations):
@@ -314,13 +318,22 @@
                 self.possibly_free_vars_for_op(op)
                 continue
             if self.can_merge_with_next_guard(op, i, operations):
-                oplist_with_guard[op.getopnum()](self, op, operations[i + 1])
+                arglocs = oplist_with_guard[op.getopnum()](self, op,
+                                                           operations[i + 1])
+                assert arglocs is not None
+                asm_operations_with_guard[op.getopnum()](self.assembler, op,
+                                                     operations[i + 1],
+                                                     arglocs, self)
                 i += 1
             elif not we_are_translated() and op.getopnum() == -124:
                 self._consider_force_spill(op)
             else:
-                oplist[op.getopnum()](self, op)
+                arglocs = oplist[op.getopnum()](self, op)
+                if arglocs is not None:
+                    asm_operations[op.getopnum()](self.assembler, op,
+                                                  arglocs, self)
             self.possibly_free_vars_for_op(op)
+            self.free_temp_vars()
             self.rm._check_invariants()
             self.fprm._check_invariants()
             i += 1
@@ -530,15 +543,17 @@
         return [loc1, res]
 
     def prepare_finish(self, op):
-        if op.numargs() > 0:
-            loc = self.loc(op.getarg(0))
-            self.possibly_free_var(op.getarg(0))
+        descr = op.getdescr()
+        fail_descr = cast_instance_to_gcref(descr)
+        # we know it does not move, but well
+        rgc._make_sure_does_not_move(fail_descr)
+        fail_descr = rffi.cast(lltype.Signed, fail_descr)
+        if op.numargs() == 1:
+            loc = self.make_sure_var_in_reg(op.getarg(0))
+            locs = [loc, imm(fail_descr)]
         else:
-            descr = op.getdescr()
-            fail_descr = cast_instance_to_gcref(descr)
-            fail_descr = rffi.cast(lltype.Signed, fail_descr)
-            loc = imm(fail_descr)
-        return [loc]
+            locs = [imm(fail_descr)]
+        return locs
 
     def prepare_call_malloc_gc(self, op):
         return self._prepare_call(op)
@@ -748,6 +763,12 @@
     prepare_getfield_raw_pure = prepare_getfield_gc
     prepare_getfield_gc_pure = prepare_getfield_gc
 
+    def prepare_increment_debug_counter(self, op):
+        args = op.getarglist()
+        base_loc = self._ensure_value_is_boxed(args[0])
+        temp_loc = self.get_scratch_reg(INT, args)
+        return [base_loc, temp_loc]
+
     def prepare_getinteriorfield_gc(self, op):
         t = unpack_interiorfielddescr(op.getdescr())
         ofs, itemsize, fieldsize, sign = t

From noreply at buildbot.pypy.org  Fri Aug 21 10:50:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 10:50:49 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-updated-backend: kill kill kill old code
Message-ID: <20150821085049.E9E761C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: ppc-updated-backend
Changeset: r79096:fbf695116cb5
Date: 2015-08-21 01:28 -0700
http://bitbucket.org/pypy/pypy/changeset/fbf695116cb5/

Log:	kill kill kill old code

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -282,7 +282,6 @@
                     ll_threadlocal_addr = rffi.cast(llmemory.Address,
                         self._debug_errno_container)
                 llop.gc_writebarrier(lltype.Void, ll_frame)
-                import pdb;pdb.set_trace()
                 ll_frame = func(ll_frame, ll_threadlocal_addr)
             finally:
                 if not self.translate_support_code:
diff --git a/rpython/jit/backend/ppc/assembler.py b/rpython/jit/backend/ppc/assembler.py
deleted file mode 100644
--- a/rpython/jit/backend/ppc/assembler.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import os
-from rpython.jit.backend.ppc import form
-#from rpython.jit.backend.ppc import asmfunc
-
-# don't be fooled by the fact that there's some separation between a
-# generic assembler class and a PPC assembler class... there's
-# certainly a RISC dependency in here, and quite possibly a PPC
-# dependency or two too.  I personally don't care :)
-
-class AssemblerException(Exception):
-    pass
-
-class Assembler(object):
-    def __init__(self):
-        self.insts = []
-        self.labels = {}
-        self.rlabels = {}
-
-    def reset(self):
-        self.insts = []
-        self.labels = {}
-        self.rlabels = {}
-
-    def label(self, name):
-        if name in self.labels:
-            raise AssemblerException, "duplicate label '%s'"%(name,)
-        self.labels[name] = len(self.insts)*4
-        self.rlabels.setdefault(len(self.insts)*4, []).append(name)
-
-    def labelname(self, base="L"):
-        i = 0
-        while 1:
-            ln = base + str(i)
-            if ln not in self.labels:
-                return ln
-            i += 1
-
-    def get_number_of_ops(self):
-        return len(self.insts)
-
-    # XXX don't need multiplication
-    def get_rel_pos(self):
-        return 4 * len(self.insts)
-
-    def patch_op(self, index):
-        last = self.insts.pop()
-        self.insts[index] = last
-
-    def assemble0(self, dump=os.environ.has_key('PPY_DEBUG')):
-        for i, inst in enumerate(self.insts):
-            for f in inst.lfields:
-                l = self.labels[inst.fields[f]] - 4*i
-                inst.fields[f] = l
-        buf = []
-        for inst in self.insts:
-            buf.append(inst)
-        if dump:
-            for i in range(len(buf)):
-                inst = self.disassemble(buf[i], self.rlabels, i*4)
-                for lab in self.rlabels.get(4*i, []):
-                    print "%s:"%(lab,)
-                print "\t%4d    %s"%(4*i, inst)
-        return buf
-
-    def assemble(self, dump=os.environ.has_key('PPY_DEBUG')):
-        c = asmfunc.AsmCode(len(self.insts)*4)
-        for i in self.insts:
-            c.emit(i)
-
-    def get_assembler_function(self):
-        c = asmfunc.AsmCode(len(self.insts)*4)
-        for i in self.insts:
-            c.emit(i)
-        return c.get_function()
-
-    def get_idescs(cls):
-        r = []
-        for name in dir(cls):
-            a = getattr(cls, name)
-            if isinstance(a, form.IDesc):
-                r.append((name, a))
-        return r
-    get_idescs = classmethod(get_idescs)
-
-    def disassemble(cls, inst, labels={}, pc=0):
-        matches = []
-        idescs = cls.get_idescs()
-        for name, idesc in idescs:
-            m = idesc.match(inst)
-            if m > 0:
-                matches.append((m, idesc, name))
-        if matches:
-            score, idesc, name = max(matches)
-            return idesc.disassemble(name, inst, labels, pc)
-    disassemble = classmethod(disassemble)
diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -2,7 +2,6 @@
 from rpython.jit.backend.ppc.ppc_form import PPCForm as Form
 from rpython.jit.backend.ppc.locations import RegisterLocation
 from rpython.jit.backend.ppc.ppc_field import ppc_fields
-from rpython.jit.backend.ppc.assembler import Assembler
 from rpython.jit.backend.ppc.arch import (IS_PPC_32, WORD, IS_PPC_64, 
                                        LR_BC_OFFSET)
 import rpython.jit.backend.ppc.register as r
@@ -54,7 +53,7 @@
 MDI = Form("rA", "rS", "sh", "mbe", "XO5", "Rc")
 MDS = Form("rA", "rS", "rB", "mbe", "XO7", "Rc")
 
-class BasicPPCAssembler(Assembler):
+class BasicPPCAssembler(object):
 
     def disassemble(cls, inst, labels={}, pc=0):
         cache = cls.__dict__.get('idesc cache')
@@ -1133,11 +1132,6 @@
         else:
             self.sld(target_reg, from_reg, numbit_reg)
 
-    def prepare_insts_blocks(self, show=False):
-        insts = self.insts
-        for inst in insts:
-            self.write32(inst)
-
     def _dump_trace(self, addr, name, formatter=-1):
         if not we_are_translated():
             if formatter != -1:
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -424,7 +424,6 @@
         mc.addi(r.SP.value, r.SP.value, frame_size)
         mc.b_abs(self.propagate_exception_path)
 
-        mc.prepare_insts_blocks()
         rawstart = mc.materialize(self.cpu, [])
         # here we do not need a function descr. This is being only called using
         # an internal ABI
@@ -526,7 +525,6 @@
         mc.addi(r.SP.value, r.SP.value, both_framesizes)
         mc.blr()
 
-        mc.prepare_insts_blocks()
         rawstart = mc.materialize(self.cpu, [])
         if IS_PPC_64:
             self.write_64_bit_func_descr(rawstart, rawstart+3*WORD)
@@ -584,7 +582,6 @@
         mc.addi(r.SP.value, r.SP.value, frame_size)
         mc.blr()
         #
-        mc.prepare_insts_blocks()
         rawstart = mc.materialize(self.cpu, [])
         self.wb_slowpath[withcards + 2 * withfloats] = rawstart
 
@@ -598,7 +595,6 @@
 
         mc.load_imm(r.RES, self.cpu.propagate_exception_descr)
         self._gen_epilogue(mc)
-        mc.prepare_insts_blocks()
         self.propagate_exception_path = mc.materialize(self.cpu, [])
 
     # The code generated here serves as an exit stub from
@@ -629,7 +625,6 @@
         # generate return and restore registers
         self._gen_epilogue(mc)
 
-        mc.prepare_insts_blocks()
         return mc.materialize(self.cpu, [], self.cpu.gc_ll_descr.gcrootmap)
 
     def _gen_epilogue(self, mc):
@@ -945,7 +940,6 @@
         frame_depth -= self.OFFSET_SPP_TO_OLD_BACKCHAIN
         mc.load_imm(r.SCRATCH, -frame_depth)
         mc.add(r.SP.value, r.SPP.value, r.SCRATCH.value)
-        mc.prepare_insts_blocks()
         mc.copy_to_raw_memory(rawstart + sp_patch_location)
 
     DESCR_REF       = 0x00
@@ -1051,7 +1045,6 @@
         return looptoken._ppc_arglocs
 
     def materialize_loop(self, looptoken, show=False):
-        self.mc.prepare_insts_blocks(show)
         self.datablockwrapper.done()
         self.datablockwrapper = None
         allblocks = self.get_asmmemmgr_blocks(looptoken)
@@ -1099,7 +1092,6 @@
                 mc = PPCBuilder()
                 offset = tok.pos_recovery_stub - tok.offset
                 mc.b_cond_offset(offset, tok.fcond)
-                mc.prepare_insts_blocks(True)
                 mc.copy_to_raw_memory(block_start + tok.offset)
             else:
                 clt.invalidate_positions.append((block_start + tok.offset,
@@ -1111,7 +1103,6 @@
         mc = PPCBuilder()
         patch_addr = faildescr._ppc_block_start + faildescr._ppc_guard_pos
         mc.b_abs(bridge_addr)
-        mc.prepare_insts_blocks()
         mc.copy_to_raw_memory(patch_addr)
         faildescr._failure_recovery_code_ofs = 0
 
diff --git a/rpython/jit/backend/ppc/rassemblermaker.py b/rpython/jit/backend/ppc/rassemblermaker.py
--- a/rpython/jit/backend/ppc/rassemblermaker.py
+++ b/rpython/jit/backend/ppc/rassemblermaker.py
@@ -61,6 +61,8 @@
     return d[name]
 
 def make_rassembler(cls):
+    # XXX tooons of very-old code patched to get medium-old code patched
+    # to get newer code :-(
     bases = [make_rassembler(b) for b in cls.__bases__]
     ns = {}
     for k, v in cls.__dict__.iteritems():
@@ -69,6 +71,6 @@
         ns[k] = v
     rcls = type('R' + cls.__name__, tuple(bases), ns)
     def emit(self, value):
-        self.insts.append(value)
+        self.write32(value)
     rcls.emit = emit
     return rcls
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -549,7 +549,7 @@
         rgc._make_sure_does_not_move(fail_descr)
         fail_descr = rffi.cast(lltype.Signed, fail_descr)
         if op.numargs() == 1:
-            loc = self.make_sure_var_in_reg(op.getarg(0))
+            loc = self._ensure_value_is_boxed(op.getarg(0))
             locs = [loc, imm(fail_descr)]
         else:
             locs = [imm(fail_descr)]

From noreply at buildbot.pypy.org  Fri Aug 21 10:50:52 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 10:50:52 +0200 (CEST)
Subject: [pypy-commit] pypy default: minor clean-ups
Message-ID: <20150821085052.0C1221C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79097:c5b725659db5
Date: 2015-08-21 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/c5b725659db5/

Log:	minor clean-ups

diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -64,12 +64,6 @@
                                               operations,
                                               original_loop_token, log=log)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU_ARM.cast_adr_to_int(adr)
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -636,8 +636,7 @@
             assert isinstance(box, Box)
             loc = self.fm.get_new_loc(box)
             locs.append(loc.value - base_ofs)
-        if looptoken.compiled_loop_token is not None:
-            # for tests
+        if looptoken.compiled_loop_token is not None:   # <- for tests
             looptoken.compiled_loop_token._ll_initial_locs = locs
 
     def can_merge_with_next_guard(self, op, i, operations):
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -100,12 +100,6 @@
         return self.assembler.assemble_bridge(faildescr, inputargs, operations,
                                               original_loop_token, log, logger)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU386.cast_adr_to_int(adr)

From noreply at buildbot.pypy.org  Fri Aug 21 11:52:23 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 11:52:23 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: pass the first test about
	mul_bridge_ovf1
Message-ID: <20150821095223.C7CEF1C1186@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79098:49ba7cf0f814
Date: 2015-08-21 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/49ba7cf0f814/

Log:	pass the first test about mul_bridge_ovf1

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -225,27 +225,14 @@
                                     call_pure_results=call_pure_results,
                                     enable_opts=enable_opts)
     try:
-        try:
-            start_state, preamble_ops = optimize_trace(metainterp_sd,
-                                                       jitdriver_sd,
-                                                       preamble_data)
-        except InvalidLoop:
-            return None
-    finally:
-        forget_optimization_info(ops)
-        forget_optimization_info(inputargs)
+        start_state, preamble_ops = optimize_trace(metainterp_sd, jitdriver_sd,
+                                                   preamble_data)
+    except InvalidLoop:
+        return None
 
-    #loop = create_empty_loop(metainterp)
-    #loop.inputargs = part.inputargs
-    #loop.operations = part.operations
-    #loop.quasi_immutable_deps = {}
-    #if part.quasi_immutable_deps:
-    #    loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
-    #lastopnum = preamble_ops[-1].getopnum()
-    #if lastopnum != rop.FINISH:
-    #if start_state is not None:
-    assert start_state is not None
-    end_label = ResOperation(rop.LABEL, start_state.end_args,
+    metainterp_sd = metainterp.staticdata
+    jitdriver_sd = metainterp.jitdriver_sd
+    end_label = ResOperation(rop.LABEL, inputargs,
                              descr=jitcell_token)
     jump_op = ResOperation(rop.JUMP, jumpargs, descr=jitcell_token)
     loop_data = UnrolledLoopData(end_label, jump_op, ops, start_state,
@@ -279,44 +266,6 @@
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
     return start_descr
-            
-            #part.quasi_immutable_deps = None
-    #         part.operations = [part.operations[-1]] + \
-    #                           [inliner.inline_op(h_ops[i]) for i in range(start, len(h_ops))] + \
-    #                           [ResOperation(rop.JUMP, [inliner.inline_arg(a) for a in jumpargs],
-    #                                         None, descr=jitcell_token)]
-    #         target_token = part.operations[0].getdescr()
-    #         assert isinstance(target_token, TargetToken)
-    #         all_target_tokens.append(target_token)
-    #         inputargs = jumpargs
-    #         jumpargs = part.operations[-1].getarglist()
-
-    #         try:
-    #             optimize_trace(metainterp_sd, jitdriver_sd, part, enable_opts,
-    #                            start_state=start_state, export_state=False)
-    #         except InvalidLoop:
-    #             return None
-
-    #         loop.operations = loop.operations[:-1] + part.operations
-    #         if part.quasi_immutable_deps:
-    #             loop.quasi_immutable_deps.update(part.quasi_immutable_deps)
-    # assert part.operations[-1].getopnum() != rop.LABEL
-
-    # if not loop.quasi_immutable_deps:
-    #     loop.quasi_immutable_deps = None
-    # for box in loop.inputargs:
-    #     assert not isinstance(box, Const)
-
-    # loop.original_jitcell_token = jitcell_token
-    # for label in all_target_tokens:
-    #     assert isinstance(label, TargetToken)
-    #     if label.virtual_state and label.short_preamble:
-    #         metainterp_sd.logger_ops.log_short_preamble([], label.short_preamble)
-    # jitcell_token.target_tokens = all_target_tokens
-    # propagate_original_jitcell_token(loop)
-    # send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
-    # record_loop_or_bridge(metainterp_sd, loop)
-    # return all_target_tokens[0]
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
@@ -324,7 +273,6 @@
     """Try to compile a new procedure by closing the current history back
     to the first operation.
     """
-    xxx
     from rpython.jit.metainterp.optimizeopt import optimize_trace
 
     history = metainterp.history
@@ -333,24 +281,21 @@
 
     loop_jitcell_token = metainterp.get_procedure_token(greenkey)
     assert loop_jitcell_token
-    assert partial_trace.operations[-1].getopnum() == rop.LABEL
 
-    part = create_empty_loop(metainterp)
-    part.inputargs = inputargs[:]
-    h_ops = history.operations
-
-    part.operations = [partial_trace.operations[-1]] + \
-                      h_ops[start:] + \
-                      [ResOperation(rop.JUMP, jumpargs, descr=loop_jitcell_token)]
-    label = part.operations[0]
-    orignial_label = label.clone()
-    assert label.getopnum() == rop.LABEL
+    end_label = ResOperation(rop.LABEL, inputargs,
+                             descr=loop_jitcell_token)
+    jump_op = ResOperation(rop.JUMP, jumpargs, descr=loop_jitcell_token)
+    enable_opts = jitdriver_sd.warmstate.enable_opts
+    ops = history.operations[start:]
+    call_pure_results = metainterp.call_pure_results
+    loop_data = UnrolledLoopData(end_label, jump_op, ops, start_state,
+                                 call_pure_results=call_pure_results,
+                                 enable_opts=enable_opts)
     try:
-        optimize_trace(metainterp_sd, jitdriver_sd, part,
-                       jitdriver_sd.warmstate.enable_opts,
-                       start_state=start_state, export_state=False)
+        loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
+                                             loop_data)
     except InvalidLoop:
-        xxx # XXX forget optimizations
+        xxx
         # Fall back on jumping to preamble
         target_token = label.getdescr()
         assert isinstance(target_token, TargetToken)
@@ -365,33 +310,29 @@
         except InvalidLoop:
             xxx # XXX forget optimizations
             return None
-    assert part.operations[-1].getopnum() != rop.LABEL
-    target_token = label.getdescr()
-    assert isinstance(target_token, TargetToken)
-    assert loop_jitcell_token.target_tokens
-    loop_jitcell_token.target_tokens.append(target_token)
-    if target_token.short_preamble:
-        metainterp_sd.logger_ops.log_short_preamble([], target_token.short_preamble)
 
     loop = partial_trace
-    loop.operations = loop.operations[:-1] + part.operations
+    target_token = TargetToken(loop_jitcell_token)
+    target_token.original_jitcell_token = loop_jitcell_token
+    target_token.short_preamble = loop_info.short_preamble
+    target_token.virtual_state = start_state.virtual_state
+    loop_ops[-1].setdescr(target_token)
+    mid_label = ResOperation(rop.LABEL, loop_info.label_args,
+                             descr=target_token)
+    loop.operations = (loop.operations + loop_info.extra_same_as + [mid_label]
+                       + loop_ops)
+    loop_jitcell_token.target_tokens.append(target_token)
 
-    quasi_immutable_deps = {}
-    if loop.quasi_immutable_deps:
-        quasi_immutable_deps.update(loop.quasi_immutable_deps)
-    if part.quasi_immutable_deps:
-        quasi_immutable_deps.update(part.quasi_immutable_deps)
-    if quasi_immutable_deps:
-        loop.quasi_immutable_deps = quasi_immutable_deps
+    #quasi_immutable_deps = {}
+    #if loop.quasi_immutable_deps:
+    #    quasi_immutable_deps.update(loop.quasi_immutable_deps)
+    #if part.quasi_immutable_deps:
+    #    quasi_immutable_deps.update(part.quasi_immutable_deps)
+    #if quasi_immutable_deps:
+    #    loop.quasi_immutable_deps = quasi_immutable_deps
 
-    for box in loop.inputargs:
-        assert isinstance(box, Box)
-
-    target_token = loop.operations[-1].getdescr()
     resumekey.compile_and_attach(metainterp, loop)
 
-    target_token = label.getdescr()
-    assert isinstance(target_token, TargetToken)
     record_loop_or_bridge(metainterp_sd, loop)
     return target_token
 
@@ -1036,20 +977,18 @@
     call_pure_results = metainterp.call_pure_results
 
     if operations[-1].getopnum() == rop.JUMP:
+        jump_op = operations[-1]
         data = BridgeCompileData(label, operations[:],
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts,
                                  inline_short_preamble=inline_short_preamble)
     else:
+        jump_op = None
         data = SimpleCompileData(label, operations[:],
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
-        try:
-            info, newops = optimize_trace(metainterp_sd, jitdriver_sd, data)
-        finally:
-            forget_optimization_info(inputargs)
-            forget_optimization_info(operations)
+        info, newops = optimize_trace(metainterp_sd, jitdriver_sd, data)
     except InvalidLoop:
         debug_print("compile_new_bridge: got an InvalidLoop")
         # XXX I am fairly convinced that optimize_bridge cannot actually raise
@@ -1058,24 +997,16 @@
         return None
 
     new_trace = create_empty_loop(metainterp)
-    new_trace.inputargs = info.inputargs
     new_trace.operations = newops
-    target_token = new_trace.operations[-1].getdescr()
-    resumekey.compile_and_attach(metainterp, new_trace)
-    record_loop_or_bridge(metainterp_sd, new_trace)
-    return target_token
-    xxxx
-    if new_trace.operations[-1].getopnum() != rop.LABEL:
-        # We managed to create a bridge.  Dispatch to resumekey to
-        # know exactly what we must do (ResumeGuardDescr/ResumeFromInterpDescr)
+    if info.final():
+        new_trace.inputargs = info.inputargs
         target_token = new_trace.operations[-1].getdescr()
         resumekey.compile_and_attach(metainterp, new_trace)
         record_loop_or_bridge(metainterp_sd, new_trace)
         return target_token
-    else:
-        raise Exception("should not occur with tracing disabled")
-        metainterp.retrace_needed(new_trace, state)
-        return None
+    new_trace.inputargs = info.renamed_inputargs
+    metainterp.retrace_needed(new_trace, info)
+    return None
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/optimizeopt/__init__.py b/rpython/jit/metainterp/optimizeopt/__init__.py
--- a/rpython/jit/metainterp/optimizeopt/__init__.py
+++ b/rpython/jit/metainterp/optimizeopt/__init__.py
@@ -58,6 +58,7 @@
         return compile_data.optimize(metainterp_sd, jitdriver_sd,
                                      optimizations, unroll)
     finally:
+        compile_data.forget_optimization_info()
         debug_stop("jit-optimize")
 
 if __name__ == '__main__':
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -27,6 +27,9 @@
     def __init__(self, inputargs):
         self.inputargs = inputargs
 
+    def final(self):
+        return True
+
 
 class Optimization(object):
     next_optimization = None
@@ -337,7 +340,7 @@
         return op
 
     def is_inputarg(self, op):
-        #return True
+        return True
         return op in self.inparg_dict
 
     def get_constant_box(self, box):
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization, LoopInfo, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
-    VirtualStateConstructor)
+    VirtualStateConstructor, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 
 class UnrollableOptimizer(Optimizer):
@@ -90,7 +90,7 @@
         self._check_no_forwarding([[start_label, end_label], ops])
         info, newops = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], ops, call_pure_results)
-        exported_state = self.export_state(start_label, end_label,
+        exported_state = self.export_state(start_label, end_label.getarglist(),
                                            info.inputargs)
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
@@ -137,25 +137,43 @@
             start_label.getarglist()[:], operations[:-1],
             call_pure_results, True)
         jump_op = operations[-1]
-        self.jump_to_existing_trace(jump_op, inline_short_preamble)
-        return info, self.optimizer._newoperations[:]
+        vs = self.jump_to_existing_trace(jump_op, inline_short_preamble)
+        if vs is None:
+            return info, self.optimizer._newoperations[:]
+        exported_state = self.export_state(start_label,
+                                           operations[-1].getarglist(),
+                                           info.inputargs)
+        self.optimizer._clean_optimization_info(self.optimizer._newoperations)
+        return exported_state, self.optimizer._newoperations
 
     def jump_to_existing_trace(self, jump_op, inline_short_preamble):
         jitcelltoken = jump_op.getdescr()
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
-        target_token = jitcelltoken.target_tokens[0]
         virtual_state = self.get_virtual_state(args)
-        target_virtual_state = target_token.virtual_state
-        
-        short_preamble = target_token.short_preamble
-        extra = self.inline_short_preamble(args,
-            short_preamble[0].getarglist(), short_preamble[1:-1],
-            short_preamble[-1].getarglist(), self.optimizer.patchguardop)
-        self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
-                                  args=args + extra,
-                                  descr=jitcelltoken.target_tokens[0]))
+        infos = [self.optimizer.getinfo(arg) for arg in args]
+        for target_token in jitcelltoken.target_tokens:
+            target_virtual_state = target_token.virtual_state
+            if target_virtual_state is None:
+                continue
+            try:
+                extra_guards = target_virtual_state.generate_guards(
+                    virtual_state, args, infos, self.optimizer.cpu)
+                assert not extra_guards.extra_guards
+            except VirtualStatesCantMatch:
+                continue
+            short_preamble = target_token.short_preamble
+            extra = self.inline_short_preamble(args,
+                short_preamble[0].getarglist(), short_preamble[1:-1],
+                short_preamble[-1].getarglist(), self.optimizer.patchguardop)
+            self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
+                                      args=args + extra,
+                                      descr=target_token))
+            return None # explicit because the return can be non-None
+        return virtual_state
 
     def filter_extra_jump_args(self, label_args, jump_args):
+        label_args = [self.get_box_replacement(x) for x in label_args]
+        jump_args = [self.get_box_replacement(x) for x in jump_args]
         new_label_args = []
         new_jump_args = []
         assert len(label_args) == len(jump_args)
@@ -190,8 +208,7 @@
             op.set_forwarded(None)
         return res
 
-    def export_state(self, start_label, end_label, renamed_inputargs):
-        original_label_args = end_label.getarglist()
+    def export_state(self, start_label, original_label_args, renamed_inputargs):
         end_args = [self.get_box_replacement(a) for a in original_label_args]
         virtual_state = self.get_virtual_state(end_args)
         inparg_mapping = [(start_label.getarg(i), end_args[i])
@@ -222,13 +239,15 @@
     def import_state(self, targetop, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
-        for source, target in exported_state.inputarg_mapping:
-            if source is not target:
-                source.set_forwarded(target)
-            info = exported_state.exported_infos.get(target, None)
+        assert len(exported_state.inputarg_mapping) == len(targetop.getarglist())
+        for i, (s, target) in enumerate(exported_state.inputarg_mapping):
+            source = targetop.getarg(i)
+            assert source is not target
+            source.set_forwarded(target)
+            info = exported_state.exported_infos.get(source, None)
             if info is not None:
                 self.optimizer.setinfo_from_preamble(source, info,
-                                        exported_state.exported_infos)
+                                            exported_state.exported_infos)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
         self.short_preamble_producer = ShortPreambleBuilder(
@@ -262,6 +281,9 @@
         self.short_preamble = short_preamble
         self.label_args = label_args
         self.extra_same_as = extra_same_as
+
+    def final(self):
+        return True
             
 class ExportedState(LoopInfo):
     """ Exported state consists of a few pieces of information:
@@ -289,3 +311,6 @@
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
         self.short_inputargs = short_inputargs
+
+    def final(self):
+        return False
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -127,6 +127,7 @@
             if box is not None:
                 fieldbox = opinfo._fields[self.fielddescrs[i].get_index()]
                 # must be there
+                xxx
                 fieldinfo = fieldbox.get_forwarded()
             else:
                 fieldbox = None
@@ -205,6 +206,7 @@
             if box is not None:
                 assert isinstance(opinfo, info.ArrayPtrInfo)
                 fieldbox = opinfo._items[i]
+                xxx
                 fieldinfo = fieldbox.get_forwarded()
             self.fieldstate[i].generate_guards(other.fieldstate[i],
                                             fieldbox, fieldinfo, state)
@@ -344,7 +346,8 @@
         if self.level == LEVEL_UNKNOWN:
             # confusingly enough, this is done also for pointers
             # which have the full range as the "bound", so it always works
-            return self._generate_guards_intbounds(other, box, extra_guards)
+            return self._generate_guards_intbounds(other, box, opinfo,
+                                                   extra_guards)
 
         # the following conditions often peek into the runtime value that the
         # box had when tracing. This value is only used as an educated guess.
@@ -412,13 +415,13 @@
                 raise VirtualStatesCantMatch("other not constant")
         assert 0, "unreachable"
 
-    def _generate_guards_intbounds(self, other, boxinfo, extra_guards):
+    def _generate_guards_intbounds(self, other, box, opinfo, extra_guards):
         if self.intbound is None:
             return
         if self.intbound.contains_bound(other.intbound):
             return
-        if (boxinfo is not None and isinstance(box, BoxInt) and
-                self.intbound.contains(box.getint())):
+        if (opinfo is not None and opinfo.is_constant() and
+                self.intbound.contains(opinfo.getint())):
             # this may generate a few more guards than needed, but they are
             # optimized away when emitting them
             self.intbound.make_guards(box, extra_guards)
@@ -502,11 +505,11 @@
             return False
         return True
 
-    def generate_guards(self, other, values, cpu):
-        assert len(self.state) == len(other.state) == len(values)
+    def generate_guards(self, other, boxes, infos, cpu):
+        assert len(self.state) == len(other.state) == len(boxes) == len(infos)
         state = GenerateGuardState(cpu)
         for i in range(len(self.state)):
-            self.state[i].generate_guards(other.state[i], values[i],
+            self.state[i].generate_guards(other.state[i], boxes[i], infos[i],
                                           state)
         return state
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -270,6 +270,7 @@
                 y -= 1
             return res
         res = self.meta_interp(f, [6, sys.maxint, 48])
+        self.check_trace_count(6)
         assert res == f(6, sys.maxint, 48)
 
     def test_loop_invariant_mul_bridge_ovf2(self):

From noreply at buildbot.pypy.org  Fri Aug 21 12:39:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 12:39:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150821103939.0F6581C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79099:aee39a70b0f5
Date: 2015-08-21 12:27 +0200
http://bitbucket.org/pypy/pypy/changeset/aee39a70b0f5/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -47,7 +47,8 @@
         self.call_pure_results = call_pure_results
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
-        from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
+        from rpython.jit.metainterp.optimizeopt.unroll import (UnrollOptimizer,
+                                                               Optimizer)
 
         if unroll:
             opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
@@ -55,7 +56,9 @@
                                          self.operations,
                                          self.call_pure_results)
         else:
-            xxx
+            opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
+            return opt.propagate_all_forward(self.start_label.getarglist(),
+               self.operations, self.call_pure_results, self.enable_opts)
 
 class SimpleCompileData(CompileData):
     """ This represents label() ops jump with no extra info associated with
@@ -242,7 +245,7 @@
         loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
                                              loop_data)
     except InvalidLoop:
-        xxx
+        return None
 
     loop = create_empty_loop(metainterp)
     loop.original_jitcell_token = jitcell_token
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -314,7 +314,7 @@
         assert res == 252
         self.check_trace_count(1)
         self.check_resops({'jump': 1, 'int_gt': 2, 'int_add': 2,
-                           'getfield_gc_pure': 1, 'int_mul': 1,
+                           'getfield_gc_pure_i': 1, 'int_mul': 1,
                            'guard_true': 2, 'int_sub': 2})
 
     def test_loops_are_transient(self):
@@ -413,7 +413,7 @@
         assert res == 42
         # CALL_PURE is not recorded in the history if all-constant args
         self.check_operations_history(int_add=0, int_mul=0,
-                                      call=0, call_pure=0)
+                                      call=0, call_pure_i=0)
 
     def test_residual_call_elidable_1(self):
         @elidable
@@ -425,7 +425,7 @@
         assert res == 42
         # CALL_PURE is recorded in the history if not-all-constant args
         self.check_operations_history(int_add=1, int_mul=0,
-                                      call=0, call_pure=1)
+                                      call=0, call_pure_i=1)
 
     def test_residual_call_elidable_2(self):
         myjitdriver = JitDriver(greens = [], reds = ['n'])
@@ -442,7 +442,7 @@
         assert res == 0
         # CALL_PURE is recorded in the history, but turned into a CALL
         # by optimizeopt.py
-        self.check_resops(call_pure=0, call=2, int_sub=0)
+        self.check_resops(call_pure_i=0, call_i=2, int_sub=0)
 
     def test_constfold_call_elidable(self):
         myjitdriver = JitDriver(greens = ['m'], reds = ['n'])
@@ -458,7 +458,7 @@
         res = self.meta_interp(f, [21, 5])
         assert res == -1
         # the CALL_PURE is constant-folded away by optimizeopt.py
-        self.check_resops(call_pure=0, call=0, int_sub=2)
+        self.check_resops(call_pure_i=0, call_i=0, int_sub=2)
 
     def test_constfold_call_elidable_2(self):
         myjitdriver = JitDriver(greens = ['m'], reds = ['n'])
@@ -478,7 +478,7 @@
         res = self.meta_interp(f, [21, 5])
         assert res == -1
         # the CALL_PURE is constant-folded away by optimizeopt.py
-        self.check_resops(call_pure=0, call=0, int_sub=2)
+        self.check_resops(call_pure_i=0, call_i=0, int_sub=2)
 
     def test_elidable_function_returning_object(self):
         myjitdriver = JitDriver(greens = ['m'], reds = ['n'])
@@ -503,7 +503,8 @@
         res = self.meta_interp(f, [21, 5])
         assert res == -1
         # the CALL_PURE is constant-folded away by optimizeopt.py
-        self.check_resops(call_pure=0, call=0, getfield_gc=1, int_sub=2)
+        self.check_resops(call_pure_r=0, call_r=0, getfield_gc_i=1, int_sub=2,
+                          call_pure_i=0, call_i=0)
 
     def test_elidable_raising(self):
         myjitdriver = JitDriver(greens = ['m'], reds = ['n'])
@@ -524,12 +525,12 @@
         res = self.meta_interp(f, [22, 6])
         assert res == -3
         # the CALL_PURE is constant-folded away during tracing
-        self.check_resops(call_pure=0, call=0, int_sub=2)
+        self.check_resops(call_pure_i=0, call_i=0, int_sub=2)
         #
         res = self.meta_interp(f, [22, -5])
         assert res == 0
         # raises: becomes CALL and is not constant-folded away
-        self.check_resops(call_pure=0, call=2, int_sub=2)
+        self.check_resops(call_pure_i=0, call_i=2, int_sub=2)
 
     def test_elidable_raising_2(self):
         myjitdriver = JitDriver(greens = ['m'], reds = ['n'])
@@ -550,12 +551,12 @@
         res = self.meta_interp(f, [22, 6])
         assert res == -3
         # the CALL_PURE is constant-folded away by optimizeopt.py
-        self.check_resops(call_pure=0, call=0, int_sub=2)
+        self.check_resops(call_pure_i=0, call_i=0, int_sub=2)
         #
         res = self.meta_interp(f, [22, -5])
         assert res == 0
         # raises: becomes CALL and is not constant-folded away
-        self.check_resops(call_pure=0, call=2, int_sub=2)
+        self.check_resops(call_pure_i=0, call_i=2, int_sub=2)
 
     def test_constant_across_mp(self):
         myjitdriver = JitDriver(greens = [], reds = ['n'])

From noreply at buildbot.pypy.org  Fri Aug 21 12:39:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 12:39:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: hack at another test
Message-ID: <20150821103941.1DF171C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79100:a94e875aab50
Date: 2015-08-21 12:39 +0200
http://bitbucket.org/pypy/pypy/changeset/a94e875aab50/

Log:	hack at another test

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -199,6 +199,39 @@
 # ____________________________________________________________
 
 
+def compile_simple_loop(metainterp, greenkey, start, inputargs, ops, jumpargs,
+                        enable_opts):
+    from rpython.jit.metainterp.optimizeopt import optimize_trace
+
+    jitdriver_sd = metainterp.jitdriver_sd
+    metainterp_sd = metainterp.staticdata
+    jitcell_token = make_jitcell_token(jitdriver_sd)
+    label = ResOperation(rop.LABEL, inputargs[:], descr=jitcell_token)
+    jump_op = ResOperation(rop.JUMP, jumpargs[:], descr=jitcell_token)
+    call_pure_results = metainterp.call_pure_results
+    data = SimpleCompileData(label, ops + [jump_op],
+                                 call_pure_results=call_pure_results,
+                                 enable_opts=enable_opts)
+    try:
+        loop_info, ops = optimize_trace(metainterp_sd, jitdriver_sd,
+                                        data)
+    except InvalidLoop:
+        return None
+    loop = create_empty_loop(metainterp)
+    loop.original_jitcell_token = jitcell_token
+    loop.inputargs = loop_info.inputargs
+    jump_op = ops[-1]
+    target_token = TargetToken(jitcell_token)
+    target_token.original_jitcell_token = jitcell_token
+    label.setdescr(target_token)
+    jump_op.setdescr(target_token)
+    loop.operations = [label] + ops
+    loop.check_consistency()
+    jitcell_token.target_tokens = [target_token]
+    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
+    record_loop_or_bridge(metainterp_sd, loop)
+    return target_token
+
 def compile_loop(metainterp, greenkey, start, inputargs, jumpargs,
                  full_preamble_needed=True, try_disabling_unroll=False):
     """Try to compile a new procedure by closing the current history back
@@ -217,12 +250,14 @@
         enable_opts = enable_opts.copy()
         del enable_opts['unroll']
 
+    ops = history.operations[start:]
+    if 'unroll' not in enable_opts:
+        return compile_simple_loop(metainterp, greenkey, start, inputargs, ops,
+                                   jumpargs, enable_opts)
     jitcell_token = make_jitcell_token(jitdriver_sd)
-    h_ops = history.operations
     label = ResOperation(rop.LABEL, inputargs,
                          descr=TargetToken(jitcell_token))
     end_label = ResOperation(rop.LABEL, jumpargs, descr=jitcell_token)
-    ops = h_ops[start:]
     call_pure_results = metainterp.call_pure_results
     preamble_data = LoopCompileData(label, end_label, ops,
                                     call_pure_results=call_pure_results,

From noreply at buildbot.pypy.org  Fri Aug 21 13:40:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 13:40:01 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: whack at this test until it
	passes
Message-ID: <20150821114001.2E7211C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79101:cab61fd68fb6
Date: 2015-08-21 13:12 +0200
http://bitbucket.org/pypy/pypy/changeset/cab61fd68fb6/

Log:	whack at this test until it passes

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -25,13 +25,12 @@
         self.optearlyforce = None
 
 class BaseTestGenerateGuards(BaseTest):
-    def guards(self, info1, info2, box_or_value, expected, inputargs=None):
-        value, box = self._box_or_value(box_or_value)
+    def guards(self, info1, info2, box, opinfo, expected, inputargs=None):
         if inputargs is None:
             inputargs = [box]
         info1.position = info2.position = 0
         state = GenerateGuardState(self.cpu)
-        info1.generate_guards(info2, value, state)
+        info1.generate_guards(info2, box, opinfo, state)
         self.compare(state.extra_guards, expected, inputargs)
 
     def compare(self, guards, expected, inputargs):
@@ -57,9 +56,7 @@
         assert not state.extra_guards
         return state
 
-    def check_invalid(self, info1, info2, box_or_value=None, state=None):
-        assert box_or_value is None
-        value, _ = None, None # self._box_or_value(box_or_value)
+    def check_invalid(self, info1, info2, box=None, opinfo=None, state=None):
         if info1.position == -1:
             info1.position = 0
         if info2.position == -1:
@@ -67,7 +64,7 @@
         if state is None:
             state = GenerateGuardState(self.cpu)
         with py.test.raises(VirtualStatesCantMatch):
-            info1.generate_guards(info2, None, None, state)
+            info1.generate_guards(info2, box, opinfo, state)
 
     def test_make_inputargs(self):
         optimizer = FakeOptimizer()
@@ -223,6 +220,8 @@
 
         constant_info = NotVirtualStateInfo(self.cpu, 'i',
                                             ConstIntBound(1))
+        constant_ptr_info = NotVirtualStateInfo(self.cpu, 'r',
+                                    info.ConstPtrInfo(ConstPtr(self.nodeaddr)))
         constclass_val = info.ConstPtrInfo(ConstPtr(self.nodeaddr))
         constclass_info = NotVirtualStateInfo(self.cpu, 'r', constclass_val)
         constclass2_info = NotVirtualStateInfo(self.cpu, 'r',
@@ -256,22 +255,33 @@
         [p0]
         guard_nonnull(p0) []
         """
-        self.guards(nonnull_info, unknown_info, unknown_val, expected)
-        self.check_invalid(nonnull_info, unknown_info, unknownnull_val)
+        nonnullbox = InputArgRef(self.nodeaddr)
+        nonnullbox2 = InputArgRef(self.node2addr)
+        knownclassopinfo = info.InstancePtrInfo(classbox1)
+        knownclass2opinfo = info.InstancePtrInfo(classbox2)
+        self.guards(nonnull_info, unknown_info, nonnullbox,
+                    None, expected)
+        self.check_invalid(nonnull_info, unknown_info, InputArgRef(), None)
         self.check_invalid(nonnull_info, unknown_info)
         self.check_invalid(nonnull_info, unknown_info)
 
         # nonnull nonnull
-        self.check_no_guards(nonnull_info, nonnull_info, nonnull_val)
-        self.check_no_guards(nonnull_info, nonnull_info, nonnull_val)
+        self.check_no_guards(nonnull_info, nonnull_info, nonnullbox, None)
+        self.check_no_guards(nonnull_info, nonnull_info, nonnullbox, None)
 
         # nonnull knownclass
-        self.check_no_guards(nonnull_info, knownclass_info, knownclass_val)
+        self.check_no_guards(nonnull_info, knownclass_info, nonnullbox,
+                             info.InstancePtrInfo(classbox1))
         self.check_no_guards(nonnull_info, knownclass_info)
 
         # nonnull constant
-        self.check_no_guards(nonnull_info, constant_info, constant_val)
-        self.check_invalid(nonnull_info, constantnull_info, constantnull_val)
+        const_nonnull = ConstPtr(self.nodeaddr)
+        const_nonnull2 = ConstPtr(self.node2addr)
+        const_null = ConstPtr(lltype.nullptr(llmemory.GCREF.TO))
+        self.check_no_guards(nonnull_info, constant_info, const_nonnull,
+                             info.ConstPtrInfo(const_nonnull))
+        self.check_invalid(nonnull_info, constantnull_info, const_null,
+                           info.ConstPtrInfo(const_null))
         self.check_no_guards(nonnull_info, constant_info)
         self.check_invalid(nonnull_info, constantnull_info)
 
@@ -281,9 +291,12 @@
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
-        self.guards(knownclass_info, unknown_info, unknown_val, expected)
-        self.check_invalid(knownclass_info, unknown_info, unknownnull_val)
-        self.check_invalid(knownclass_info, unknown_info, knownclass2_val)
+        self.guards(knownclass_info, unknown_info, InputArgRef(self.nodeaddr),
+                    None, expected)
+        self.check_invalid(knownclass_info, unknown_info, InputArgRef(), None)
+        self.check_invalid(knownclass_info, unknown_info,
+                           InputArgRef(self.node2addr),
+                           info.InstancePtrInfo(classbox2))
         self.check_invalid(knownclass_info, unknown_info)
         self.check_invalid(knownclass_info, unknown_info)
         self.check_invalid(knownclass_info, unknown_info)
@@ -293,20 +306,26 @@
         [p0]
         guard_class(p0, ConstClass(node_vtable)) []
         """
-        self.guards(knownclass_info, nonnull_info, knownclass_val, expected)
-        self.check_invalid(knownclass_info, nonnull_info, knownclass2_val)
+        self.guards(knownclass_info, nonnull_info, InputArgRef(self.nodeaddr),
+                    None, expected)
+        self.check_invalid(knownclass_info, nonnull_info,
+                           InputArgRef(self.node2addr), None)
         self.check_invalid(knownclass_info, nonnull_info)
         self.check_invalid(knownclass_info, nonnull_info)
 
         # knownclass knownclass
-        self.check_no_guards(knownclass_info, knownclass_info, knownclass_val)
-        self.check_invalid(knownclass_info, knownclass2_info, knownclass2_val)
+        self.check_no_guards(knownclass_info, knownclass_info,
+                             nonnullbox, knownclassopinfo)
+        self.check_invalid(knownclass_info, knownclass2_info,
+                           nonnullbox2, knownclass2opinfo)
         self.check_no_guards(knownclass_info, knownclass_info)
         self.check_invalid(knownclass_info, knownclass2_info)
 
         # knownclass constant
-        self.check_invalid(knownclass_info, constantnull_info, constantnull_val)
-        self.check_invalid(knownclass_info, constclass2_info, constclass2_val)
+        self.check_invalid(knownclass_info, constantnull_info,
+                           const_null, info.ConstPtrInfo(const_null))
+        self.check_invalid(knownclass_info, constclass2_info, const_nonnull2,
+                           info.ConstPtrInfo(const_nonnull2))
         self.check_invalid(knownclass_info, constantnull_info)
         self.check_invalid(knownclass_info, constclass2_info)
 
@@ -316,8 +335,9 @@
         [i0]
         guard_value(i0, 1) []
         """
-        self.guards(constant_info, unknown_info, constant_val, expected)
-        self.check_invalid(constant_info, unknown_info, unknownnull_val)
+        self.guards(constant_info, unknown_info, InputArgInt(1),
+                    ConstIntBound(1), expected)
+        self.check_invalid(constant_info, unknown_info, InputArgRef(), None)
         self.check_invalid(constant_info, unknown_info)
         self.check_invalid(constant_info, unknown_info)
 
@@ -326,24 +346,29 @@
         [i0]
         guard_value(i0, 1) []
         """
-        self.guards(constant_info, nonnull_info, constant_val, expected)
-        self.check_invalid(constant_info, nonnull_info, constclass2_val)
+        self.guards(constant_info, nonnull_info, ConstInt(1),
+                    ConstIntBound(1), expected)
+        self.check_invalid(constant_info, nonnull_info,
+                           ConstInt(3), ConstIntBound(3))
         self.check_invalid(constant_info, nonnull_info)
         self.check_invalid(constant_info, nonnull_info)
 
         # constant knownclass
         expected = """
-        [i0]
-        guard_value(i0, 1) []
+        [p0]
+        guard_value(p0, ConstPtr(nodeaddr)) []
         """
-        self.guards(constant_info, knownclass_info, constant_val, expected)
-        self.check_invalid(constant_info, knownclass_info, unknownnull_val)
+        self.guards(constant_ptr_info, knownclass_info,
+                    const_nonnull, info.ConstPtrInfo(const_nonnull), expected)
+        self.check_invalid(constant_info, knownclass_info, InputArgRef())
         self.check_invalid(constant_info, knownclass_info)
         self.check_invalid(constant_info, knownclass_info)
 
         # constant constant
-        self.check_no_guards(constant_info, constant_info, constant_val)
-        self.check_invalid(constant_info, constantnull_info, constantnull_val)
+        self.check_no_guards(constant_info, constant_info,
+                             ConstInt(1), ConstIntBound(1))
+        self.check_invalid(constant_info, constantnull_info,
+                           const_null, info.ConstPtrInfo(const_null))
         self.check_no_guards(constant_info, constant_info)
         self.check_invalid(constant_info, constantnull_info)
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -420,8 +420,7 @@
             return
         if self.intbound.contains_bound(other.intbound):
             return
-        if (opinfo is not None and opinfo.is_constant() and
-                self.intbound.contains(opinfo.getint())):
+        if (box is not None and self.intbound.contains(box.getint())):
             # this may generate a few more guards than needed, but they are
             # optimized away when emitting them
             self.intbound.make_guards(box, extra_guards)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -372,6 +372,10 @@
     def copy_value_from(self, other):
         self.setint(other.getint())
 
+    def constbox(self):
+        from rpython.jit.metainterp import history
+        return history.ConstInt(self.getint())
+
     def nonnull(self):
         return self._resint != 0
 
@@ -394,6 +398,10 @@
     def copy_value_from(self, other):
         self.setfloatstorage(other.getfloatstorage())
 
+    def constbox(self):
+        from rpython.jit.metainterp import history
+        return history.ConstFloat(self.getfloatstorage())
+
     def nonnull(self):
         return bool(longlong.extract_bits(self._resfloat))
 
@@ -428,6 +436,11 @@
     def nonnull(self):
         return bool(self._resref)
 
+    def constbox(self):
+        from rpython.jit.metainterp import history
+        return history.ConstPtr(self.getref_base())
+
+
 class AbstractInputArg(AbstractValue):
     _forwarded = None
 

From noreply at buildbot.pypy.org  Fri Aug 21 13:40:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 13:40:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: whack at this test until it
	passes
Message-ID: <20150821114003.55F321C034E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79102:6d7007ea2e79
Date: 2015-08-21 13:13 +0200
http://bitbucket.org/pypy/pypy/changeset/6d7007ea2e79/

Log:	whack at this test until it passes

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -374,11 +374,11 @@
 
 
     def test_intbounds(self):
-        value1 = IntOptValue(BoxInt(15))
-        value1.intbound.make_ge(IntBound(0, 10))
-        value1.intbound.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(IntOptValue(BoxInt()))
+        value1 = IntBound()
+        value1.make_ge(IntBound(0, 10))
+        value1.make_le(IntBound(20, 30))
+        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
+        info2 = NotVirtualStateInfo(self.cpu, 'i', IntBound())
         expected = """
         [i0]
         i1 = int_ge(i0, 0)

From noreply at buildbot.pypy.org  Fri Aug 21 13:49:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 13:49:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: port some more tests
Message-ID: <20150821114937.559BF1C034E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79103:57470787557f
Date: 2015-08-21 13:49 +0200
http://bitbucket.org/pypy/pypy/changeset/57470787557f/

Log:	port some more tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -374,11 +374,11 @@
 
 
     def test_intbounds(self):
-        value1 = IntBound()
+        value1 = IntUnbounded()
         value1.make_ge(IntBound(0, 10))
         value1.make_le(IntBound(20, 30))
         info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
-        info2 = NotVirtualStateInfo(self.cpu, 'i', IntBound())
+        info2 = NotVirtualStateInfo(self.cpu, 'i', IntUnbounded())
         expected = """
         [i0]
         i1 = int_ge(i0, 0)
@@ -386,39 +386,38 @@
         i2 = int_le(i0, 30)
         guard_true(i2) []
         """
-        self.guards(info1, info2, value1, expected)
-        self.check_invalid(info1, info2, BoxInt(50))
+        self.guards(info1, info2, InputArgInt(), value1, expected)
+        self.check_invalid(info1, info2, InputArgInt(50))
 
     def test_intbounds_constant(self):
-        value1 = IntOptValue(BoxInt(15))
-        value1.intbound.make_ge(IntBound(0, 10))
-        value1.intbound.make_le(IntBound(20, 30))
-        info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(IntOptValue(ConstInt(10000)))
+        value1 = IntUnbounded()
+        value1.make_ge(IntBound(0, 10))
+        value1.make_le(IntBound(20, 30))
+        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
+        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(10000))
         self.check_invalid(info1, info2)
-        info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(IntOptValue(ConstInt(11)))
+        info1 = NotVirtualStateInfo(self.cpu, 'i', value1)
+        info2 = NotVirtualStateInfo(self.cpu, 'i', ConstIntBound(11))
         self.check_no_guards(info1, info2)
 
     def test_known_class(self):
-        value1 = PtrOptValue(self.nodebox)
-        classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value1.make_constant_class(None, classbox)
-        info1 = NotVirtualStateInfo(value1)
-        info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        value1 = info.InstancePtrInfo(classbox)
+        info1 = NotVirtualStateInfo(self.cpu, 'r', value1)
+        info2 = NotVirtualStateInfo(self.cpu, 'r', None)
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []        
         """
-        self.guards(info1, info2, self.nodebox, expected)
-        self.check_invalid(info1, info2, BoxPtr())
+        self.guards(info1, info2, InputArgRef(self.nodeaddr), None, expected)
+        self.check_invalid(info1, info2, InputArgRef())
 
     def test_known_class_value(self):
-        value1 = PtrOptValue(self.nodebox)
-        classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value1.make_constant_class(None, classbox)
-        box = self.nodebox
-        guards = value1.make_guards(box)
+        classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        value1 = info.InstancePtrInfo(classbox)
+        box = InputArgRef()
+        guards = []
+        value1.make_guards(box, guards)
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []
@@ -426,10 +425,10 @@
         self.compare(guards, expected, [box])
 
     def test_known_value(self):
-        value1 = PtrOptValue(self.nodebox)
-        value1.make_constant(ConstInt(1))
-        box = self.nodebox
-        guards = value1.make_guards(box)
+        value1 = ConstIntBound(1)
+        box = InputArgInt()
+        guards = []
+        value1.make_guards(box, guards)
         expected = """
         [i0]
         guard_value(i0, 1) []
@@ -437,21 +436,20 @@
         self.compare(guards, expected, [box])
 
     def test_equal_inputargs(self):
-        value = PtrOptValue(self.nodebox)
-        classbox = self.cpu.ts.cls_of_box(self.nodebox)
-        value.make_constant_class(None, classbox)
-        knownclass_info = NotVirtualStateInfo(value)
+        classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        value = info.InstancePtrInfo(classbox)
+        knownclass_info = NotVirtualStateInfo(self.cpu, 'r', value)
         vstate1 = VirtualState([knownclass_info, knownclass_info])
         assert vstate1.generalization_of(vstate1)
 
-        unknown_info1 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
         vstate2 = VirtualState([unknown_info1, unknown_info1])
         assert vstate2.generalization_of(vstate2)
         assert not vstate1.generalization_of(vstate2)
         assert vstate2.generalization_of(vstate1)
 
-        unknown_info1 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
-        unknown_info2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        unknown_info1 = NotVirtualStateInfo(self.cpu, 'r', None)
+        unknown_info2 = NotVirtualStateInfo(self.cpu, 'r', None)
         vstate3 = VirtualState([unknown_info1, unknown_info2])
         assert vstate3.generalization_of(vstate2)
         assert vstate3.generalization_of(vstate1)
@@ -462,24 +460,25 @@
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
-        state = vstate1.generate_guards(vstate2, [value, value], self.cpu)
-        self.compare(state.extra_guards, expected, [self.nodebox])
+        box = InputArgRef(self.nodeaddr)
+        state = vstate1.generate_guards(vstate2, [box, box], [None, None],
+                                        self.cpu)
+        self.compare(state.extra_guards, expected, [box])
 
         with py.test.raises(VirtualStatesCantMatch):
-            vstate1.generate_guards(vstate3, [value, value],
+            vstate1.generate_guards(vstate3, [box, box], [None, None],
                                     self.cpu)
         with py.test.raises(VirtualStatesCantMatch):
-            vstate2.generate_guards(vstate3, [value, value],
+            vstate2.generate_guards(vstate3, [box, box], [None, None],
                                     self.cpu)
 
 
     def test_generate_guards_on_virtual_fields_matches_array(self):
-        innervalue1 = PtrOptValue(self.nodebox)
-        constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(innervalue1)
+        classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(classbox)
+        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
         innerinfo2.position = 1
 
         descr = object()
@@ -490,14 +489,15 @@
         info2 = VArrayStateInfo(descr)
         info2.fieldstate = [innerinfo2]
 
-        value1 = VArrayValue(descr, None, 1, self.nodebox)
-        value1._items[0] = PtrOptValue(self.nodebox)
+        value1 = info.ArrayPtrInfo(vdescr=descr, size=1)
+        box = InputArgRef(self.nodeaddr)
+        value1._items[0] = box
 
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
-        self.guards(info1, info2, value1, expected, [self.nodebox])
+        self.guards(info1, info2, InputArgRef(), value1, expected, [box])
 
     def test_generate_guards_on_virtual_fields_matches_instance(self):
         innervalue1 = PtrOptValue(self.nodebox)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -21,6 +21,11 @@
         self.msg = msg
         self.state = state
 
+def get_forwarded(box):
+    if not isinstance(box, Const):
+        return box.get_forwarded()
+    xxx
+
 class GenerateGuardState(object):
     def __init__(self, cpu=None, guards=None, renum=None, bad=None):
         self.cpu = cpu
@@ -206,8 +211,7 @@
             if box is not None:
                 assert isinstance(opinfo, info.ArrayPtrInfo)
                 fieldbox = opinfo._items[i]
-                xxx
-                fieldinfo = fieldbox.get_forwarded()
+                fieldinfo = get_forwarded(fieldbox)
             self.fieldstate[i].generate_guards(other.fieldstate[i],
                                             fieldbox, fieldinfo, state)
 
@@ -309,6 +313,8 @@
             self.constbox = info.getconst()
             if type == 'r':
                 self.known_class = info.get_known_class(cpu)
+            elif type == 'i':
+                self.intbound = info
         elif type == 'r':
             if info:
                 self.known_class = info.get_known_class(cpu)

From noreply at buildbot.pypy.org  Fri Aug 21 13:57:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 13:57:45 +0200 (CEST)
Subject: [pypy-commit] pypy default: improve loading
Message-ID: <20150821115745.43BDD1C034E@cobra.cs.uni-duesseldorf.de>

Author: fijal
Branch: 
Changeset: r79104:23592a8bf7e2
Date: 2015-08-21 13:58 +0200
http://bitbucket.org/pypy/pypy/changeset/23592a8bf7e2/

Log:	improve loading

diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -52,6 +52,14 @@
 static int opened_profile(char *interp_name);
 static void flush_codes(void);
 
+#ifdef __APPLE__
+#define UNWIND_NAME "/usr/lib/system/libunwind.dylib"
+#define UNW_PREFIX "unw"
+#else
+#define UNWIND_NAME "libunwind.so"
+#define UNW_PREFIX "_ULx86_64"
+#endif
+
 RPY_EXTERN
 char *vmprof_init(int fd, double interval, char *interp_name)
 {
@@ -62,15 +70,15 @@
     if (!unw_get_reg) {
         void *libhandle;
 
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
+        if (!(libhandle = dlopen(UNWIND_NAME, RTLD_LAZY | RTLD_LOCAL)))
             goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
+        if (!(unw_get_reg = dlsym(libhandle, UNW_PREFIX "_get_reg")))
             goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
+        if (!(unw_get_proc_info = dlsym(libhandle, UNW_PREFIX "_get_proc_info")))
             goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
+        if (!(unw_init_local = dlsym(libhandle, UNW_PREFIX  "_init_local")))
             goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
+        if (!(unw_step = dlsym(libhandle, UNW_PREFIX  "_step")))
             goto error;
     }
     if (prepare_concurrent_bufs() < 0)

From noreply at buildbot.pypy.org  Fri Aug 21 14:10:41 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:10:41 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: reverted some changes
Message-ID: <20150821121041.9BD381C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79105:8956d04bc00d
Date: 2015-08-21 10:45 +0200
http://bitbucket.org/pypy/pypy/changeset/8956d04bc00d/

Log:	reverted some changes

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -569,7 +569,7 @@
         fullsize = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(original_loop_token)
-        faildescr.bridge_rawstart = rawstart
+        original_loop_token.rawstart = rawstart
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         debug_bridge(descr_number, rawstart, codeendpos)
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -202,16 +202,14 @@
             vl.inputargs = version.inputargs
             vl.operations = version.operations
             vl.original_jitcell_token = jitcell_token
-            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
-                                             faildescr, version.inputargs,
-                                             version.operations, jitcell_token)
+            send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+                                   faildescr, version.inputargs,
+                                   version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
-            version.compiled = faildescr.bridge_rawstart
-            assert asminfo is not None
         # stitch the rest of the traces
         for version in loop.versions:
             for faildescr in version.faildescrs[1:]:
-                cpu.stitch_bridge(faildescr, version.compiled)
+                cpu.stitch_bridge(faildescr, jitcell_token)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
@@ -443,7 +441,6 @@
     #if metainterp_sd.warmrunnerdesc is not None:    # for tests
     #    metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(
     #        original_loop_token)
-    return asminfo
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -52,8 +52,6 @@
         #
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars, opt.has_two_labels)
         gso.propagate_all_forward(opt.loop, user_code)
-        # connect all compile loop version fail descriptors to this version
-        #version.register_all_guards(loop.operations, opt.appended_arg_count)
         #
         #
         end = time.clock()
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -221,7 +221,7 @@
         assert res == f(60,58.4547) == 58.4547
 
     def test_accum(self):
-        myjitdriver = JitDriver(greens = [], reds = 'auto')
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
         T = lltype.Array(rffi.DOUBLE)
         def f(d, value):
             va = lltype.malloc(T, d, flavor='raw', zero=True)
@@ -239,7 +239,7 @@
                 i += 1
             lltype.free(va, flavor='raw')
             return r
-        res = self.meta_interp(f, [60,0.5], vec_all=True)
+        res = self.meta_interp(f, [60,0.5], vec=True)
         assert res == f(60,0.5) == 60*0.5
 
 

From noreply at buildbot.pypy.org  Fri Aug 21 14:10:43 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:10:43 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: finally the simplest version
 working again. Accidently mixed up the order of the fail args which caused
 to result in wrong register mapping
Message-ID: <20150821121043.C1AEF1C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79106:dd5a2c07e944
Date: 2015-08-21 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/dd5a2c07e944/

Log:	finally the simplest version working again. Accidently mixed up the
	order of the fail args which caused to result in wrong register
	mapping

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -758,6 +758,7 @@
         label = operations[idx]
         self.label_pos = idx
         self.inputargs = label.getarglist()
+        self.renamed_inputargs = label.getarglist()
         self.compiled = None
 
     def register_guard(self, op):
@@ -767,6 +768,9 @@
         assert isinstance(descr, CompileLoopVersionDescr)
         descr.version = self
         self.faildescrs.append(descr)
+        # note: stitching a guard must resemble the order of the label
+        # otherwise a wrong mapping is handed to the register allocator
+        op.setfailargs(self.renamed_inputargs)
 
     def update_token(self, jitcell_token, all_target_tokens):
         # this is only invoked for versioned loops!
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -493,6 +493,9 @@
             return
         if vector:
             # add accumulation info to the descriptor
+            for version in self.loop.versions:
+                # this needs to be done for renamed (accum arguments)
+                version.renamed_inputargs = [ renamer.rename_map.get(arg,arg) for arg in version.inputargs ]
             self.appended_arg_count = len(sched_data.invariant_vector_vars)
             for guard_node in self.dependency_graph.guards:
                 op = guard_node.getoperation()

From noreply at buildbot.pypy.org  Fri Aug 21 14:10:46 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:10:46 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: test_vectorize passes again
Message-ID: <20150821121046.094EA1C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79107:97f9f17c7fc2
Date: 2015-08-21 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/97f9f17c7fc2/

Log:	test_vectorize passes again

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -536,7 +536,7 @@
             self.cpu.profile_agent.native_code_written(name,
                                                        rawstart, full_size)
         return AsmInfo(ops_offset, rawstart + looppos,
-                       size_excluding_failure_stuff - looppos)
+                       size_excluding_failure_stuff - looppos, rawstart)
 
     @rgc.no_release_gil
     def assemble_bridge(self, faildescr, inputargs, operations,
@@ -569,7 +569,6 @@
         fullsize = self.mc.get_relative_pos()
         #
         rawstart = self.materialize_loop(original_loop_token)
-        original_loop_token.rawstart = rawstart
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
         debug_bridge(descr_number, rawstart, codeendpos)
@@ -590,11 +589,11 @@
             name = "Bridge # %s" % (descr_number,)
             self.cpu.profile_agent.native_code_written(name,
                                                        rawstart, fullsize)
-        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
+        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
 
     def stitch_bridge(self, faildescr, target):
         assert target != 0
-        self.patch_jump_for_descr(faildescr, target)
+        self.patch_jump_for_descr(faildescr, target.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -202,14 +202,17 @@
             vl.inputargs = version.inputargs
             vl.operations = version.operations
             vl.original_jitcell_token = jitcell_token
-            send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                    faildescr, version.inputargs,
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
+            assert asminfo is not None
+            version.compiled = asminfo
         # stitch the rest of the traces
         for version in loop.versions:
             for faildescr in version.faildescrs[1:]:
-                cpu.stitch_bridge(faildescr, jitcell_token)
+                if faildescr.version.compiled:
+                    cpu.stitch_bridge(faildescr, version.compiled)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
@@ -441,6 +444,7 @@
     #if metainterp_sd.warmrunnerdesc is not None:    # for tests
     #    metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(
     #        original_loop_token)
+    return asminfo
 
 # ____________________________________________________________
 
@@ -746,11 +750,6 @@
     def loop_version(self):
         return True
 
-    def clone(self):
-        cloned = ResumeGuardDescr.clone(self)
-        cloned.version = self.version
-        return cloned
-
 class AllVirtuals:
     llopaque = True
     cache = None
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -746,7 +746,6 @@
                 pass_by -= 1
     return -1
 
-
 class LoopVersion(object):
 
     def __init__(self, operations):
@@ -766,8 +765,8 @@
         assert isinstance(op, GuardResOp)
         descr = op.getdescr()
         assert isinstance(descr, CompileLoopVersionDescr)
+        self.faildescrs.append(descr)
         descr.version = self
-        self.faildescrs.append(descr)
         # note: stitching a guard must resemble the order of the label
         # otherwise a wrong mapping is handed to the register allocator
         op.setfailargs(self.renamed_inputargs)
@@ -873,12 +872,18 @@
         return None
 
     def snapshot(self):
-        version = LoopVersion(self.copy_operations())
+        faildescrs = []
+        version = LoopVersion(self.copy_operations(faildescrs))
+        version.faildescrs = faildescrs
+        if not we_are_translated():
+            print "LOOP SNAPSHOT"
+            for op in version.operations:
+                print "", op
         self.versions.append(version)
         return version
 
-    def copy_operations(self):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr, CompileLoopVersionDescr
+    def copy_operations(self, faildescrs=None):
+        from rpython.jit.metainterp.compile import ResumeGuardDescr
         ignore = (rop.DEBUG_MERGE_POINT,)
         operations = []
         for op in self.operations:
@@ -890,8 +895,8 @@
             if cloned.is_guard() and descr:
                 assert isinstance(descr, ResumeGuardDescr)
                 cloned.setdescr(descr.clone())
-                if isinstance(descr, CompileLoopVersionDescr):
-                    descr.version.register_guard(cloned)
+                if faildescrs and descr.loop_version():
+                    faildescrs.append(cloned.getdescr())
         return operations
 
     def get_display_text(self):    # for graphpage.py
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -272,10 +272,10 @@
             if not op.is_guard():
                 continue
             descr = op.getdescr()
-            if isinstance(descr, CompileLoopVersionDescr):
+            if descr.loop_version():
                 version.register_guard(op)
 
-        if user_code and False:
+        if user_code:
             version = loop.snapshot()
             self.eliminate_array_bound_checks(loop, version)
 
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -233,8 +233,8 @@
             # in this case a guard k <= d is inserted which fails right away!
             while i < d:
                 myjitdriver.jit_merge_point()
-                #if not(i < k):
-                #    k -= 1
+                if not(i < k):
+                    k -= 1
                 r += va[i]
                 i += 1
             lltype.free(va, flavor='raw')
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1004,11 +1004,13 @@
     ops_offset - dict of offsets of operations or None
     asmaddr - (int) raw address of assembler block
     asmlen - assembler block length
+    rawstart - address a guard can jump to
     """
-    def __init__(self, ops_offset, asmaddr, asmlen):
+    def __init__(self, ops_offset, asmaddr, asmlen, rawstart):
         self.ops_offset = ops_offset
         self.asmaddr = asmaddr
         self.asmlen = asmlen
+        self.rawstart = rawstart
 
 class JitDebugInfo(object):
     """ An object representing debug info. Attributes meanings:

From noreply at buildbot.pypy.org  Fri Aug 21 14:10:48 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:10:48 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: finished the refactoring. the root
 loop has now it's own version,
 which makes generate_pending_loop_versions easier to understand. a fail
 descr now always points to the loop version (field version) it would like
 to jump to
Message-ID: <20150821121048.19FA61C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79108:90d4590adf3c
Date: 2015-08-21 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/90d4590adf3c/

Log:	finished the refactoring. the root loop has now it's own version,
	which makes generate_pending_loop_versions easier to understand. a
	fail descr now always points to the loop version (field version) it
	would like to jump to handle not compiled versions (because they are
	not needed) gracefully

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -162,7 +162,7 @@
 
     if loop.versions is not None:
         # every different loop version must update their target tokens
-        for version in loop.versions:
+        for version in loop.versions[1:]:
             version.update_token(jitcell_token, all_target_tokens)
 
     if not loop.quasi_immutable_deps:
@@ -194,10 +194,11 @@
     if loop.versions is not None:
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
-        for version in loop.versions:
-            if len(version.faildescrs) == 0:
+        root = loop.versions[0]
+        for faildescr in root.faildescrs:
+            version = faildescr.version
+            if not version or version.compiled():
                 continue
-            faildescr = version.faildescrs[0]
             vl = create_empty_loop(metainterp)
             vl.inputargs = version.inputargs
             vl.operations = version.operations
@@ -207,12 +208,19 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version.compiled = asminfo
+            version._compiled = asminfo
+            faildescr.version = None
         # stitch the rest of the traces
-        for version in loop.versions:
-            for faildescr in version.faildescrs[1:]:
-                if faildescr.version.compiled:
-                    cpu.stitch_bridge(faildescr, version.compiled)
+        for lv in loop.versions:
+            if not lv.compiled():
+                # the version was never compiled, do not bother
+                # to assign it's fail descr
+                continue
+            for faildescr in lv.faildescrs:
+                version = faildescr.version
+                if version and version.compiled():
+                    cpu.stitch_bridge(faildescr, version._compiled)
+                faildescr.version = None
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -748,28 +748,62 @@
 
 class LoopVersion(object):
 
-    def __init__(self, operations):
-        self.operations = operations
+    def __init__(self, loop):
         self.faildescrs = []
-        #
-        idx = index_of_first(rop.LABEL, operations)
-        assert idx >= 0
-        label = operations[idx]
-        self.label_pos = idx
-        self.inputargs = label.getarglist()
-        self.renamed_inputargs = label.getarglist()
-        self.compiled = None
+        self._compiled = None
+        if loop:
+            self.operations = self.copy_operations(loop.operations) 
+            idx = index_of_first(rop.LABEL, self.operations)
+            assert idx >= 0
+            label = self.operations[idx]
+            self.inputargs = label.getarglist()
+            self.renamed_inputargs = label.getarglist()
+        else:
+            self.operations = None
+            self.inputargs = None
+            self.renamed_inputargs = None
 
-    def register_guard(self, op):
+    def compiled(self):
+        if self.operations is None:
+            # root version must always be compiled
+            return True
+
+        return self._compiled is not None
+
+    def copy_operations(self, operations):
+        from rpython.jit.metainterp.compile import ResumeGuardDescr
+        ignore = (rop.DEBUG_MERGE_POINT,)
+        oplist = []
+        for op in operations:
+            if op.getopnum() in ignore:
+                continue
+            cloned = op.clone()
+            oplist.append(cloned)
+            if cloned.is_guard():
+                olddescr = cloned.getdescr()
+                if not olddescr:
+                    continue
+                descr = olddescr.clone()
+                cloned.setdescr(descr)
+                if olddescr.loop_version():
+                    # copy the version
+                    descr.version = olddescr.version
+                    self.faildescrs.append(descr)
+        return oplist
+
+    def register_guard(self, op, version):
         from rpython.jit.metainterp.compile import CompileLoopVersionDescr
         assert isinstance(op, GuardResOp)
         descr = op.getdescr()
+        if not descr.loop_version():
+            assert 0, "cannot register a guard that is not a CompileLoopVersionDescr"
         assert isinstance(descr, CompileLoopVersionDescr)
+        descr.version = version
         self.faildescrs.append(descr)
-        descr.version = self
         # note: stitching a guard must resemble the order of the label
         # otherwise a wrong mapping is handed to the register allocator
-        op.setfailargs(self.renamed_inputargs)
+        op.setfailargs(version.renamed_inputargs)
+        assert version.renamed_inputargs is not None
 
     def update_token(self, jitcell_token, all_target_tokens):
         # this is only invoked for versioned loops!
@@ -798,7 +832,6 @@
         label.setdescr(token)
         jump.setdescr(token)
 
-
 class TreeLoop(object):
     inputargs = None
     operations = None
@@ -872,9 +905,11 @@
         return None
 
     def snapshot(self):
-        faildescrs = []
-        version = LoopVersion(self.copy_operations(faildescrs))
-        version.faildescrs = faildescrs
+        if len(self.versions) == 0:
+            # create a root version, simplyfies the code in compile.py
+            self.versions.append(LoopVersion(None))
+        root_version = self.versions[0]
+        version = LoopVersion(self)
         if not we_are_translated():
             print "LOOP SNAPSHOT"
             for op in version.operations:
@@ -882,23 +917,6 @@
         self.versions.append(version)
         return version
 
-    def copy_operations(self, faildescrs=None):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        ignore = (rop.DEBUG_MERGE_POINT,)
-        operations = []
-        for op in self.operations:
-            if op.getopnum() in ignore:
-                continue
-            cloned = op.clone()
-            operations.append(cloned)
-            descr = cloned.getdescr()
-            if cloned.is_guard() and descr:
-                assert isinstance(descr, ResumeGuardDescr)
-                cloned.setdescr(descr.clone())
-                if faildescrs and descr.loop_version():
-                    faildescrs.append(cloned.getdescr())
-        return operations
-
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
 
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -152,8 +152,7 @@
         assert operations[self.index] is self.op
         operations[self.index] = None
         descr = self.op.getdescr()
-        if isinstance(descr, CompileLoopVersionDescr) and descr.version:
-            descr.version.faildescrs.remove(descr)
+        if descr and descr.loop_version():
             descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None
@@ -265,19 +264,20 @@
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
         #
-        assert len(loop.versions) == 1, "none or more than one version created"
-        version = loop.versions[0]
+        assert len(loop.versions) == 2, "need one orignal loop version (before trans) and the current one"
+        root_version = loop.versions[0]
+        version = loop.versions[1]
 
         for op in loop.operations:
             if not op.is_guard():
                 continue
             descr = op.getdescr()
             if descr.loop_version():
-                version.register_guard(op)
+                root_version.register_guard(op, version)
 
         if user_code:
             version = loop.snapshot()
-            self.eliminate_array_bound_checks(loop, version)
+            self.eliminate_array_bound_checks(loop, root_version, version)
 
     def emit_operation(self, op):
         self.renamer.rename(op)
@@ -286,7 +286,7 @@
     def operation_position(self):
         return len(self._newoperations)
 
-    def eliminate_array_bound_checks(self, loop, version):
+    def eliminate_array_bound_checks(self, loop, root_version, version):
         self._newoperations = []
         for key, guards in self.strongest_guards.items():
             if len(guards) <= 1:
@@ -299,7 +299,7 @@
                 transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
                     other.set_to_none(loop.operations)
-                    version.register_guard(transitive_guard)
+                    root_version.register_guard(transitive_guard, version)
 
         if self.has_two_labels:
             oplist = [loop.operations[0]] + self._newoperations + \
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -38,6 +38,7 @@
     user_code = not jitdriver_sd.vec and warmstate.vec_all
     if user_code and user_loop_bail_fast_path(loop, warmstate):
         return
+    # the original loop (output of optimize_unroll)
     version = loop.snapshot()
     try:
         debug_start("vec-opt-loop")
@@ -493,7 +494,7 @@
             return
         if vector:
             # add accumulation info to the descriptor
-            for version in self.loop.versions:
+            for version in self.loop.versions[1:]:
                 # this needs to be done for renamed (accum arguments)
                 version.renamed_inputargs = [ renamer.rename_map.get(arg,arg) for arg in version.inputargs ]
             self.appended_arg_count = len(sched_data.invariant_vector_vars)
@@ -874,7 +875,7 @@
                 box = result
             elif accum.operator == Accum.MULTIPLY:
                 # multiply is only supported by floats
-                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstFloat(1.0)], box)
+                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstFloat(1.0), ConstInt(size)], box)
                 sched_data.invariant_oplist.append(op)
             else:
                 raise NotImplementedError("can only handle + and *")

From noreply at buildbot.pypy.org  Fri Aug 21 14:17:52 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:17:52 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: added missing parameter needed for
 llgraph to the test suite
Message-ID: <20150821121752.55B0A1C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79109:21e4d477dc9c
Date: 2015-08-21 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/21e4d477dc9c/

Log:	added missing parameter needed for llgraph to the test suite

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -191,7 +191,7 @@
     """
     metainterp_sd = metainterp.staticdata
     cpu = metainterp_sd.cpu
-    if loop.versions is not None:
+    if loop.versions:
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
         root = loop.versions[0]
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -264,16 +264,17 @@
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
         #
-        assert len(loop.versions) == 2, "need one orignal loop version (before trans) and the current one"
-        root_version = loop.versions[0]
-        version = loop.versions[1]
+        if len(loop.versions) >= 2:
+            assert len(loop.version) == 2
+            root_version = loop.versions[0]
+            version = loop.versions[1]
 
-        for op in loop.operations:
-            if not op.is_guard():
-                continue
-            descr = op.getdescr()
-            if descr.loop_version():
-                root_version.register_guard(op, version)
+            for op in loop.operations:
+                if not op.is_guard():
+                    continue
+                descr = op.getdescr()
+                if descr.loop_version():
+                    root_version.register_guard(op, version)
 
         if user_code:
             version = loop.snapshot()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -184,7 +184,7 @@
         v10[i64|2] = vec_box(2)
         v20[i64|2] = vec_int_pack(v10[i64|2], i0, 0, 1)
         v30[i64|2] = vec_int_pack(v20[i64|2], i1, 1, 1)
-        v40[i64|2] = vec_int_expand(73)
+        v40[i64|2] = vec_int_expand(73,2)
         #
         v50[i64|2] = vec_int_add(v30[i64|2], v40[i64|2])
         """, False)
@@ -200,7 +200,7 @@
         v10[f64|2] = vec_box(2)
         v20[f64|2] = vec_float_pack(v10[f64|2], f0, 0, 1)
         v30[f64|2] = vec_float_pack(v20[f64|2], f1, 1, 1)
-        v40[f64|2] = vec_float_expand(73.0)
+        v40[f64|2] = vec_float_expand(73.0,2)
         #
         v50[f64|2] = vec_float_add(v30[f64|2], v40[f64|2])
         """, False)
@@ -220,7 +220,7 @@
         v10[f64|2] = vec_box(2)
         v20[f64|2] = vec_float_pack(v10[f64|2], f0, 0, 1)
         v30[f64|2] = vec_float_pack(v20[f64|2], f1, 1, 1)
-        v40[f64|2] = vec_float_expand(f5) # only expaned once
+        v40[f64|2] = vec_float_expand(f5,2) # only expaned once
         #
         v50[f64|2] = vec_float_add(v30[f64|2], v40[f64|2])
         v60[f64|2] = vec_float_add(v50[f64|2], v40[f64|2])
@@ -367,7 +367,7 @@
         pack3 = self.pack(loop1, 4, 6, I64, None)
         loop2 = self.schedule(loop1, [pack1,pack2,pack3], prepend_invariant=True)
         loop3 = self.parse("""
-        v9[i64|2] = vec_int_expand(255)
+        v9[i64|2] = vec_int_expand(255,2)
         v10[i64|2] = vec_raw_load(p0, i1, 2, descr=long)
         v11[i64|2] = vec_int_and(v10[i64|2], v9[i64|2])
         guard_true(v11[i64|2]) []
@@ -404,8 +404,8 @@
         pack1 = self.pack(loop1, 0, 2, I64, I64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[i64|2] = vec_int_expand(255)
-        v2[i64|2] = vec_int_expand(i1)
+        v1[i64|2] = vec_int_expand(255,2)
+        v2[i64|2] = vec_int_expand(i1,2)
         v3[i64|2] = vec_int_and(v1[i64|2], v2[i64|2])
         """, False)
         self.assert_equal(loop2, loop3)
@@ -418,8 +418,8 @@
         pack1 = self.pack(loop1, 0, 2, I64, I64)
         loop2 = self.schedule(loop1, [pack1], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[i64|2] = vec_int_expand(255)
-        v2[i64|2] = vec_int_expand(i1)
+        v1[i64|2] = vec_int_expand(255, 2)
+        v2[i64|2] = vec_int_expand(i1, 2)
         v3[i64|2] = vec_int_and(v1[i64|2], v2[i64|2])
         """, False)
         self.assert_equal(loop2, loop3)
@@ -437,7 +437,7 @@
         pack4 = self.pack(loop1, 4, 6, I64, I64)
         loop2 = self.schedule(loop1, [pack1,pack4], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[i64|2] = vec_int_expand(255)
+        v1[i64|2] = vec_int_expand(255,2)
         v2[i64|2] = vec_box(2)
         v3[i64|2] = vec_int_pack(v2[i64|2], i1, 0, 1)
         v4[i64|2] = vec_int_pack(v3[i64|2], i2, 1, 1)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -134,7 +134,7 @@
         opt.combine_packset()
         opt.schedule(True)
         if with_guard_opt:
-            gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
+            gso = GuardStrengthenOpt(opt.dependency_graph.index_vars, opt.has_two_labels)
             gso.propagate_all_forward(opt.loop)
         return opt
 
@@ -147,7 +147,7 @@
         opt.schedule(True)
         if not opt.costmodel.profitable():
             raise NotAProfitableLoop()
-        gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
+        gso = GuardStrengthenOpt(opt.dependency_graph.index_vars, opt.has_two_labels)
         gso.propagate_all_forward(opt.loop)
         return opt
 
@@ -988,7 +988,7 @@
         opt="""
         [p0,i0]
         label(p0,i0)
-        v3 = vec_int_expand(42)
+        v3 = vec_int_expand(42, 2)
         label(p0,i0,v3)
         i20 = int_add(i0, 1)
         i30 = int_lt(i20, 10)
@@ -1018,7 +1018,7 @@
         opt="""
         [p0,i0,f3]
         label(p0,i0,f3)
-        v3 = vec_float_expand(f3)
+        v3 = vec_float_expand(f3,2)
         label(p0,i0,f3,v3)
         i20 = int_add(i0, 1)
         i30 = int_lt(i20, 10)
@@ -1412,7 +1412,7 @@
         guard_false(i33, descr=) [p1, p0, p6, p7, i29, None, None]
         jump(p0, p1, p6, p7, i29, p14, p15)
         """        
-        opt = self.schedule(self.parse_loop(trace))
+        #opt = self.schedule(self.parse_loop(trace))
         #self.debug_print_operations(opt.loop)
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):

From noreply at buildbot.pypy.org  Fri Aug 21 14:50:10 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:50:10 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: implemented
 (g/s)etarrayitem_(gc/raw)[_pure] for vector operations. the assembler was
 missing
Message-ID: <20150821125010.AFAD41C0557@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79110:dcfb609c4b12
Date: 2015-08-21 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/dcfb609c4b12/

Log:	implemented (g/s)etarrayitem_(gc/raw)[_pure] for vector operations.
	the assembler was missing added a test case to verify

diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -138,6 +138,8 @@
         self._vec_load(resloc, src_addr, integer_loc.value,
                        size_loc.value, aligned_loc.value)
 
+    genop_vec_getarrayitem_gc = genop_vec_getarrayitem_raw
+
     def genop_vec_raw_load(self, op, arglocs, resloc):
         base_loc, ofs_loc, size_loc, ofs, integer_loc, aligned_loc = arglocs
         src_addr = addr_add(base_loc, ofs_loc, ofs.value, 0)
@@ -164,6 +166,8 @@
         self._vec_store(dest_loc, value_loc, integer_loc.value,
                         size_loc.value, aligned_loc.value)
 
+    genop_discard_vec_setarrayitem_gc = genop_discard_vec_setarrayitem_raw
+
     def genop_discard_vec_raw_store(self, op, arglocs):
         base_loc, ofs_loc, value_loc, size_loc, baseofs, integer_loc, aligned_loc = arglocs
         dest_loc = addr_add(base_loc, ofs_loc, baseofs.value, 0)
@@ -560,6 +564,7 @@
         self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs),
                           imm(integer), imm(aligned)], result_loc)
 
+    consider_vec_getarrayitem_gc = consider_vec_getarrayitem_raw
     consider_vec_raw_load = consider_vec_getarrayitem_raw
 
     def consider_vec_setarrayitem_raw(self, op):
@@ -578,6 +583,7 @@
         self.perform_discard(op, [base_loc, ofs_loc, value_loc,
                                  imm(itemsize), imm(ofs), imm(integer), imm(aligned)])
 
+    consider_vec_setarrayitem_gc = consider_vec_setarrayitem_raw
     consider_vec_raw_store = consider_vec_setarrayitem_raw
 
     def consider_vec_arith(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -265,7 +265,7 @@
         self.eliminate_guards(loop)
         #
         if len(loop.versions) >= 2:
-            assert len(loop.version) == 2
+            assert len(loop.versions) == 2
             root_version = loop.versions[0]
             version = loop.versions[1]
 
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -807,6 +807,10 @@
     rop.RAW_LOAD:         rop.VEC_RAW_LOAD,
     rop.GETARRAYITEM_RAW: rop.VEC_GETARRAYITEM_RAW,
     rop.GETARRAYITEM_GC: rop.VEC_GETARRAYITEM_GC,
+    # note that there is no _PURE operation for vector operations.
+    # reason: currently we do not care if it is pure or not!
+    rop.GETARRAYITEM_RAW_PURE: rop.VEC_GETARRAYITEM_RAW,
+    rop.GETARRAYITEM_GC_PURE: rop.VEC_GETARRAYITEM_GC,
     rop.RAW_STORE:        rop.VEC_RAW_STORE,
     rop.SETARRAYITEM_RAW: rop.VEC_SETARRAYITEM_RAW,
     rop.SETARRAYITEM_GC: rop.VEC_SETARRAYITEM_GC,
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -283,6 +283,27 @@
         res = self.meta_interp(f, [i])
         assert res == f(i)
 
+    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3)])
+    def test_list_vectorize(self,i,v1,v2):
+        myjitdriver = JitDriver(greens = [],
+                                reds = 'auto')
+        def f(d, v1, v2):
+            a = [v1] * d
+            b = [v2] * d
+            i = 0
+            while i < len(a):
+                myjitdriver.jit_merge_point()
+                a[i] = a[i] + b[i]
+                i += 1
+            s = 0
+            for i in range(d):
+                s += a[i]
+            return s
+        res = self.meta_interp(f, [i,v1,v2], vec_all=True)
+        # sum helps to generate the rounding error of floating points
+        # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
+        assert res == f(i,v1,v2) == sum([v1+v2]*i)
+
 class VectorizeLLtypeTests(VectorizeTests):
     pass
 

From noreply at buildbot.pypy.org  Fri Aug 21 14:56:58 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:56:58 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: removed debug output
Message-ID: <20150821125658.D83481C0557@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79111:f40c1f7b277b
Date: 2015-08-21 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/f40c1f7b277b/

Log:	removed debug output

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -910,10 +910,6 @@
             self.versions.append(LoopVersion(None))
         root_version = self.versions[0]
         version = LoopVersion(self)
-        if not we_are_translated():
-            print "LOOP SNAPSHOT"
-            for op in version.operations:
-                print "", op
         self.versions.append(version)
         return version
 

From noreply at buildbot.pypy.org  Fri Aug 21 14:57:01 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 14:57:01 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: translation issue
Message-ID: <20150821125701.0FDB11C0557@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79112:95a341896bd5
Date: 2015-08-21 14:54 +0200
http://bitbucket.org/pypy/pypy/changeset/95a341896bd5/

Log:	translation issue

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -276,9 +276,9 @@
                 if descr.loop_version():
                     root_version.register_guard(op, version)
 
-        if user_code:
-            version = loop.snapshot()
-            self.eliminate_array_bound_checks(loop, root_version, version)
+            if user_code:
+                version = loop.snapshot()
+                self.eliminate_array_bound_checks(loop, root_version, version)
 
     def emit_operation(self, op):
         self.renamer.rename(op)

From noreply at buildbot.pypy.org  Fri Aug 21 15:00:34 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 15:00:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: start passing some tests on darwin
Message-ID: <20150821130034.E77CA1C0557@cobra.cs.uni-duesseldorf.de>

Author: fijal
Branch: 
Changeset: r79113:2f729fff5878
Date: 2015-08-21 14:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2f729fff5878/

Log:	start passing some tests on darwin

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -83,6 +83,15 @@
     func.c_name = cont_name
     func._dont_inline_ = True
 
+    if sys.platform == 'darwin':
+        # according to internet "At the time UNIX was written in 1974...."
+        # "... all C functions are prefixed with _"
+        cont_name = '_' + cont_name
+        tramp_name = '_' + tramp_name
+        PLT = ""
+    else:
+        PLT = "@PLT"
+
     assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
         "rvmprof only supports x86-64 CPUs for now")
 
@@ -107,17 +116,15 @@
     target.write("""\
 \t.text
 \t.globl\t%(tramp_name)s
-\t.type\t%(tramp_name)s, @function
 %(tramp_name)s:
 \t.cfi_startproc
 \tpushq\t%(reg)s
 \t.cfi_def_cfa_offset 16
-\tcall %(cont_name)s at PLT
+\tcall %(cont_name)s%(PLT)s
 \taddq\t$8, %%rsp
 \t.cfi_def_cfa_offset 8
 \tret
 \t.cfi_endproc
-\t.size\t%(tramp_name)s, .-%(tramp_name)s
 """ % locals())
 
     def tok2cname(tok):
@@ -151,7 +158,7 @@
     )
 
     return rffi.llexternal(
-        tramp_name,
+        tramp_name.lstrip("_"),
         [token2lltype(tok) for tok in token] + [lltype.Signed],
         token2lltype(restok),
         compilation_info=eci,
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -181,6 +181,7 @@
             ('all', '$(DEFAULT_TARGET)', []),
             ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Fri Aug 21 15:00:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 15:00:37 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix vmprof on OS X
Message-ID: <20150821130037.265E21C0557@cobra.cs.uni-duesseldorf.de>

Author: fijal
Branch: 
Changeset: r79114:6acadde07a0d
Date: 2015-08-21 14:49 +0200
http://bitbucket.org/pypy/pypy/changeset/6acadde07a0d/

Log:	fix vmprof on OS X

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -79,6 +79,7 @@
 
     cont_name = 'rpyvmprof_f_%s_%s' % (name, token)
     tramp_name = 'rpyvmprof_t_%s_%s' % (name, token)
+    orig_tramp_name = tramp_name
 
     func.c_name = cont_name
     func._dont_inline_ = True
@@ -136,7 +137,7 @@
 
     header = 'RPY_EXTERN %s %s(%s);\n' % (
         tok2cname(restok),
-        tramp_name,
+        orig_tramp_name,
         ', '.join([tok2cname(tok) for tok in token] + ['long']))
 
     header += """\
@@ -150,7 +151,7 @@
 #endif
 #define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s
 }
-""" % (tramp_name, tramp_name, tramp_name)
+""" % (tramp_name, orig_tramp_name, tramp_name)
 
     eci = ExternalCompilationInfo(
         post_include_bits = [header],
@@ -158,7 +159,7 @@
     )
 
     return rffi.llexternal(
-        tramp_name.lstrip("_"),
+        orig_tramp_name,
         [token2lltype(tok) for tok in token] + [lltype.Signed],
         token2lltype(restok),
         compilation_info=eci,

From noreply at buildbot.pypy.org  Fri Aug 21 15:00:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 15:00:39 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix for OS X
Message-ID: <20150821130039.341471C0557@cobra.cs.uni-duesseldorf.de>

Author: fijal
Branch: 
Changeset: r79115:8ea80c0d6f77
Date: 2015-08-21 15:00 +0200
http://bitbucket.org/pypy/pypy/changeset/8ea80c0d6f77/

Log:	fix for OS X

diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -461,7 +461,11 @@
     close(srcfd);
 #else
     // freebsd and mac
+#if defined(__APPLE__)
+	sprintf(buf, "vmmap %d", getpid());
+#else
     sprintf(buf, "procstat -v %d", getpid());
+#endif
     FILE *srcf = popen(buf, "r");
     if (!srcf)
         return -1;

From noreply at buildbot.pypy.org  Fri Aug 21 15:33:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 15:33:35 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi 1.2.1 as released
Message-ID: <20150821133335.464A91C0063@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79116:32db8f8cf274
Date: 2015-08-21 15:33 +0200
http://bitbucket.org/pypy/pypy/changeset/32db8f8cf274/

Log:	Update to cffi 1.2.1 as released

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.0
+Version: 1.2.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.0"
+VERSION = "1.2.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,4 +3427,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -160,6 +160,35 @@
     assert func.name == 'sin'
     assert func.BType == ', ), , False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -238,6 +267,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,6 +25,9 @@
     if 1:     # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
+    else:
+        kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+                                      ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -318,15 +318,32 @@
                 import cffi
                 ffi = cffi.FFI()
                 ffi.set_source("pack3.mymod", "/*code would be here*/")
+                ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
+                from distutils.command.build_ext import build_ext
+                import os
+
+                class TestBuildExt(build_ext):
+                    def pre_run(self, ext, ffi):
+                        print('_make_setuptools_api: in pre_run:', end=" ")
+                        assert ffi._hi_there == 42
+                        assert ext.name == "pack3.mymod"
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print('creating %r' % (fn,))
+                        open(fn, 'w').close()
+
                 setup(name='example1',
                       version='0.1',
                       packages=['pack3'],
                       package_dir={'': 'src1'},
-                      cffi_modules=["src1/pack3/_build.py:ffi"])
+                      cffi_modules=["src1/pack3/_build.py:ffi"],
+                      cmdclass={'build_ext': TestBuildExt},
+                      )
             """)
 
     @chdir_to_tmp
@@ -335,6 +352,7 @@
         self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None}}})
 
@@ -344,6 +362,7 @@
         self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None,
                                                       'mymod.SO': None}}})

From noreply at buildbot.pypy.org  Fri Aug 21 15:43:18 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 15:43:18 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: documentation additions (command
 line flags), added description of the ABC optimization, note on limitations
Message-ID: <20150821134318.225521C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79117:3c01987639c7
Date: 2015-08-21 15:43 +0200
http://bitbucket.org/pypy/pypy/changeset/3c01987639c7/

Log:	documentation additions (command line flags), added description of
	the ABC optimization, note on limitations

diff --git a/rpython/doc/jit/index.rst b/rpython/doc/jit/index.rst
--- a/rpython/doc/jit/index.rst
+++ b/rpython/doc/jit/index.rst
@@ -25,6 +25,7 @@
    pyjitpl5
    optimizer
    virtualizable
+   vectorization
 
 - :doc:`Overview `: motivating our approach
 
diff --git a/rpython/doc/jit/optimizer.rst b/rpython/doc/jit/optimizer.rst
--- a/rpython/doc/jit/optimizer.rst
+++ b/rpython/doc/jit/optimizer.rst
@@ -178,6 +178,10 @@
 It is prepended to all optimizations and thus extends the Optimizer class
 and unrolls the loop once before it proceeds.
 
+Vectorization
+-------------
+
+- :doc:`Vectorization `
 
 What is missing from this document
 ----------------------------------
diff --git a/rpython/doc/jit/vectorization.rst b/rpython/doc/jit/vectorization.rst
--- a/rpython/doc/jit/vectorization.rst
+++ b/rpython/doc/jit/vectorization.rst
@@ -7,6 +7,18 @@
 that is that they use the same index variable and offset can be expressed as a
 a linear or affine combination.
 
+Command line flags:
+
+* --jit vec=1: turns on the vectorization for marked jitdrivers
+  (e.g. those in the NumPyPy module).
+* --jit vec_all=1: turns on the vectorization for any jit driver. See parameters for
+  the filtering heuristics of traces.
+* --jit vec_ratio=2: A number from 0 to 10 that represents a real number (vec_ratio / 10).
+  This filters traces if vec_all is enabled. N is the trace count then the number of
+  vector transformable operations (add_int -> vec_add_int) M, the following must hold:
+  M / N >= (vec_ratio / 10)
+* --jit vec_length=60: The maximum number of trace instructions the vectorizer filters for.
+
 Features
 --------
 
@@ -38,6 +50,28 @@
 load/store instructions) are not removed. The backend removes these instructions
 while assembling the trace.
 
+In addition a simple heuristic (enabled by --jit vec_all=1) tries to remove
+array bound checks for application level loops. It tries to identify the array
+bound checks and adds a transitive guard at the top of the loop::
+
+    label(...)
+    ...
+    guard(i < n) # index guard
+    ...
+    guard(i < len(a))
+    a = load(..., i, ...)
+    ...
+    jump(...)
+    # becomes
+    guard(n < len(a))
+    label(...)
+    guard(i < n) # index guard
+    ...
+    a = load(..., i, ...)
+    ...
+    jump(...)
+
+
 
 Future Work and Limitations
 ---------------------------
@@ -54,5 +88,9 @@
   to have 2 xmm registers (one filled with zero bits and the other with one every bit).
   This cuts down 2 instructions for guard checking, trading for higher register pressure.
 * prod, sum are only supported by 64 bit data types
+* isomorphic function prevents the following cases for combination into a pair:
+  1) getarrayitem_gc, getarrayitem_gc_pure
+  2) int_add(v,1), int_sub(v,-1)
 
 .. _PMUL: http://stackoverflow.com/questions/8866973/can-long-integer-routines-benefit-from-sse/8867025#8867025
+

From noreply at buildbot.pypy.org  Fri Aug 21 15:54:05 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 15:54:05 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: translation issues and added
 documentation to some routines in the scheduler
Message-ID: <20150821135405.A68A11C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79118:81c2f4a28f89
Date: 2015-08-21 15:54 +0200
http://bitbucket.org/pypy/pypy/changeset/81c2f4a28f89/

Log:	translation issues and added documentation to some routines in the
	scheduler

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -592,7 +592,7 @@
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
 
     def stitch_bridge(self, faildescr, target):
-        assert target != 0
+        assert target.rawstart != 0
         self.patch_jump_for_descr(faildescr, target.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -771,7 +771,8 @@
         return self._compiled is not None
 
     def copy_operations(self, operations):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
+        from rpython.jit.metainterp.compile import (ResumeGuardDescr,
+                CompileLoopVersionDescr)
         ignore = (rop.DEBUG_MERGE_POINT,)
         oplist = []
         for op in operations:
@@ -787,6 +788,8 @@
                 cloned.setdescr(descr)
                 if olddescr.loop_version():
                     # copy the version
+                    assert isinstance(olddescr, CompileLoopVersionDescr)
+                    assert isinstance(descr, CompileLoopVersionDescr)
                     descr.version = olddescr.version
                     self.faildescrs.append(descr)
         return oplist
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -303,6 +303,7 @@
 
 
     def transform_pack(self):
+        """ high level transformation routine of a pack to operations """
         op = self.pack.leftmost()
         args = op.getarglist()
         self.before_argument_transform(args)
@@ -344,6 +345,7 @@
         return self.pack.operations
 
     def transform_arguments(self, args):
+        """ transforming one argument to a vector box argument """
         for i,arg in enumerate(args):
             if isinstance(arg, BoxVector):
                 continue
@@ -444,6 +446,7 @@
         return vbox_cloned
 
     def unpack(self, vbox, index, count, arg_ptype):
+        """ extract parts of the vector box into another vector box """
         assert index < vbox.getcount()
         assert index + count <= vbox.getcount()
         assert count > 0
@@ -491,6 +494,9 @@
         assert result.getcount() > arg0.getcount()
 
     def expand(self, arg, argidx):
+        """ expand a value into a vector box. useful for arith metic
+            of one vector with a scalar (either constant/varialbe)
+        """
         elem_count = self.input_type.getcount()
         vbox = self.input_type.new_vector_box(elem_count)
         box_type = arg.type
@@ -665,10 +671,6 @@
     def get_input_type_given(self, output_type, op):
         raise AssertionError("cannot infer input type from output type")
 
-    # OLD
-    def determine_output_type(self, op):
-        return None
-
 GUARD_TF = PassThroughOp((PT_INT_GENERIC,))
 INT_OP_TO_VOP = OpToVectorOp((PT_INT_GENERIC, PT_INT_GENERIC), INT_RES)
 FLOAT_OP_TO_VOP = OpToVectorOp((PT_FLOAT_GENERIC, PT_FLOAT_GENERIC), FLOAT_RES)
@@ -784,6 +786,7 @@
             scheduler.oplist.append(op)
 
     def as_vector_operation(self, scheduler, pack):
+        """ transform a pack into a single or several operation """
         assert pack.opcount() > 1
         # properties that hold for the pack are:
         # + isomorphism (see func above)
@@ -794,7 +797,6 @@
         op = pack.operations[0].getoperation()
         determine_trans(op).as_vector_operation(pack, self, scheduler, oplist)
         #
-        # XXX
         if pack.is_accumulating():
             box = oplist[position].result
             assert box is not None
@@ -804,6 +806,9 @@
                 scheduler.renamer.start_renaming(op.result, box)
 
     def unpack_from_vector(self, op, scheduler):
+        """ if a box is needed that is currently stored within a vector
+            box, this utility creates a unpacking instruction
+        """
         args = op.getarglist()
 
         # unpack for an immediate use
@@ -917,6 +922,7 @@
             node.pack_position = i
 
     def rightmost_match_leftmost(self, other):
+        """ check if pack A can be combined with pack B """
         assert isinstance(other, Pack)
         rightmost = self.operations[-1]
         leftmost = other.operations[0]
@@ -955,6 +961,7 @@
                    self.right is other.right
 
 class AccumPair(Pair):
+    """ A pair that keeps track of an accumulation value """
     def __init__(self, left, right, input_type, output_type, accum):
         assert isinstance(left, Node)
         assert isinstance(right, Node)
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -283,7 +283,7 @@
         res = self.meta_interp(f, [i])
         assert res == f(i)
 
-    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3)])
+    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3), (15,44.0,22.2)])
     def test_list_vectorize(self,i,v1,v2):
         myjitdriver = JitDriver(greens = [],
                                 reds = 'auto')

From noreply at buildbot.pypy.org  Fri Aug 21 15:59:26 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 21 Aug 2015 15:59:26 +0200 (CEST)
Subject: [pypy-commit] pypy no_boehm_libdl: remove libdl from linker for
	boehm gc
Message-ID: <20150821135926.4CCD81C0063@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: no_boehm_libdl
Changeset: r79119:b84a79056d18
Date: 2015-08-21 16:59 +0300
http://bitbucket.org/pypy/pypy/changeset/b84a79056d18/

Log:	remove libdl from linker for boehm gc

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -854,7 +854,7 @@
         platform = None
     else:
         library_dir = ''
-        libraries = ['gc', 'dl']
+        libraries = ['gc']
         includes=['gc/gc.h']
     eci = ExternalCompilationInfo(
         platform=platform,

From noreply at buildbot.pypy.org  Fri Aug 21 16:05:20 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 16:05:20 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: docu
Message-ID: <20150821140520.297061C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79120:e7a3de7993aa
Date: 2015-08-21 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/e7a3de7993aa/

Log:	docu

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -12,6 +12,7 @@
 class SchedulerData(object):
     pass
 class Scheduler(object):
+    """ The base class to be instantiated to (re)schedule a vector trace. """
     def __init__(self, graph, sched_data):
         assert isinstance(sched_data, SchedulerData)
         self.graph = graph
@@ -37,6 +38,8 @@
         return -1
 
     def schedulable(self, candidate):
+        """ Is the candidate scheduleable? Boils down to dependency_count == 0
+        """
         if candidate.pack:
             pack = candidate.pack
             if pack.is_accumulating():
@@ -52,6 +55,11 @@
         return candidate.depends_count() == 0
 
     def scheduled(self, node):
+        """ Call this function if an operation has been emitted
+            adds new operations to the schedule list if
+            their dependency count drops to zero.
+            In addition it keeps the list sorted (see priority)
+        """
         node.position = len(self.oplist)
         for dep in node.provides()[:]: # COPY
             to = dep.to
@@ -79,6 +87,9 @@
         node.emitted = True
 
     def emit_into(self, oplist, renamer, unpack=False):
+        """ Emit all the operations into the oplist parameter.
+            Initiates the scheduling.
+        """
         self.renamer = renamer
         self.oplist = oplist
         self.unpack = unpack
@@ -303,7 +314,7 @@
 
 
     def transform_pack(self):
-        """ high level transformation routine of a pack to operations """
+        """ High level transformation routine of a pack to operations """
         op = self.pack.leftmost()
         args = op.getarglist()
         self.before_argument_transform(args)
@@ -345,7 +356,7 @@
         return self.pack.operations
 
     def transform_arguments(self, args):
-        """ transforming one argument to a vector box argument """
+        """ Transforming one argument to a vector box argument """
         for i,arg in enumerate(args):
             if isinstance(arg, BoxVector):
                 continue
@@ -446,7 +457,7 @@
         return vbox_cloned
 
     def unpack(self, vbox, index, count, arg_ptype):
-        """ extract parts of the vector box into another vector box """
+        """ Extract parts of the vector box into another vector box """
         assert index < vbox.getcount()
         assert index + count <= vbox.getcount()
         assert count > 0
@@ -494,7 +505,7 @@
         assert result.getcount() > arg0.getcount()
 
     def expand(self, arg, argidx):
-        """ expand a value into a vector box. useful for arith metic
+        """ Expand a value into a vector box. useful for arith metic
             of one vector with a scalar (either constant/varialbe)
         """
         elem_count = self.input_type.getcount()
@@ -637,10 +648,9 @@
         return None
 
 class StoreToVectorStore(OpToVectorOp):
-    """
-    Storing operations are special because they are not allowed
-    to store to memory if the vector is not fully filled.
-    Thus a modified split_pack function
+    """ Storing operations are special because they are not allowed
+        to store to memory if the vector is not fully filled.
+        Thus a modified split_pack function.
     """
     def __init__(self):
         OpToVectorOp.__init__(self, (None, None, PT_GENERIC), None)
@@ -659,8 +669,8 @@
 
 class PassThroughOp(OpToVectorOp):
     """ This pass through is only applicable if the target
-    operation is capable of handling vector operations.
-    Guard true/false is such an example.
+        operation is capable of handling vector operations.
+        Guard true/false is such an example.
     """
     def __init__(self, args):
         OpToVectorOp.__init__(self, args, None)
@@ -720,12 +730,12 @@
 
 def determine_input_output_types(pack, node, forward):
     """ This function is two fold. If moving forward, it
-    gets an input type from the packs output type and returns
-    the transformed packtype.
+        gets an input type from the packs output type and returns
+        the transformed packtype.
 
-    Moving backward, the origins pack input type is the output
-    type and the transformation of the packtype (in reverse direction)
-    is the input
+        Moving backward, the origins pack input type is the output
+        type and the transformation of the packtype (in reverse direction)
+        is the input
     """
     op = node.getoperation()
     op2vecop = determine_trans(op)
@@ -763,8 +773,8 @@
         self.seen = {}
 
     def schedule_candidate(self, scheduler, candidate):
-        """ if you implement a scheduler this operations is called
-        to emit the actual operation into the oplist of the scheduler
+        """ If you implement a scheduler this operations is called
+            to emit the actual operation into the oplist of the scheduler.
         """
         renamer = scheduler.renamer
         if candidate.pack:
@@ -786,7 +796,9 @@
             scheduler.oplist.append(op)
 
     def as_vector_operation(self, scheduler, pack):
-        """ transform a pack into a single or several operation """
+        """ Transform a pack into a single or several operation.
+            Calls the as_vector_operation of the OpToVectorOp implementation.
+        """
         assert pack.opcount() > 1
         # properties that hold for the pack are:
         # + isomorphism (see func above)
@@ -806,8 +818,8 @@
                 scheduler.renamer.start_renaming(op.result, box)
 
     def unpack_from_vector(self, op, scheduler):
-        """ if a box is needed that is currently stored within a vector
-            box, this utility creates a unpacking instruction
+        """ If a box is needed that is currently stored within a vector
+            box, this utility creates a unpacking instruction.
         """
         args = op.getarglist()
 
@@ -881,8 +893,8 @@
         return self.operations[0].getoperation()
 
     def is_full(self, vec_reg_size):
-        """ if one input element times the opcount is equal
-        to the vector register size, we are full!
+        """ If one input element times the opcount is equal
+            to the vector register size, we are full!
         """
         ptype = self.input_type
         if self.input_type is None:
@@ -922,7 +934,7 @@
             node.pack_position = i
 
     def rightmost_match_leftmost(self, other):
-        """ check if pack A can be combined with pack B """
+        """ Check if pack A can be combined with pack B """
         assert isinstance(other, Pack)
         rightmost = self.operations[-1]
         leftmost = other.operations[0]
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -33,6 +33,9 @@
 
 def optimize_vector(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, warmstate):
+    """ Enter the world of SIMD instructions. Bails if it cannot
+        transform the trace.
+    """
     optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations,
                     inline_short_preamble, start_state, False)
     user_code = not jitdriver_sd.vec and warmstate.vec_all
@@ -88,8 +91,9 @@
             raise
 
 def user_loop_bail_fast_path(loop, warmstate):
-    """ in a fast path over the trace loop: try to prevent vecopt
-    of spending time on a loop that will most probably fail """
+    """ In a fast path over the trace loop: try to prevent vecopt
+        of spending time on a loop that will most probably fail.
+    """
 
     resop_count = 0 # the count of operations minus debug_merge_points
     vector_instr = 0
@@ -112,7 +116,7 @@
     if resop_count > warmstate.vec_length:
         return True
 
-    if float(vector_instr)/float(resop_count) <= warmstate.vec_ratio:
+    if (float(vector_instr)/float(resop_count)) < warmstate.vec_ratio:
         return True
 
     return False

From noreply at buildbot.pypy.org  Fri Aug 21 17:19:55 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 17:19:55 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: docu vectorize.py
Message-ID: <20150821151955.1DD5E1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79121:571344b0097d
Date: 2015-08-21 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/571344b0097d/

Log:	docu vectorize.py

diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -299,12 +299,13 @@
         return unroll_count-1 # it is already unrolled once
 
     def find_adjacent_memory_refs(self):
-        """ the pre pass already builds a hash of memory references and the
-        operations. Since it is in SSA form there are no array indices.
-        If there are two array accesses in the unrolled loop
-        i0,i1 and i1 = int_add(i0,c), then i0 = i0 + 0, i1 = i0 + 1.
-        They are represented as a linear combination: i*c/d + e, i is a variable,
-        all others are integers that are calculated in reverse direction"""
+        """ The pre pass already builds a hash of memory references and the
+            operations. Since it is in SSA form there are no array indices.
+            If there are two array accesses in the unrolled loop
+            i0,i1 and i1 = int_add(i0,c), then i0 = i0 + 0, i1 = i0 + 1.
+            They are represented as a linear combination: i*c/d + e, i is a variable,
+            all others are integers that are calculated in reverse direction
+        """
         loop = self.loop
         operations = loop.operations
 
@@ -328,6 +329,9 @@
                             self.packset.add_pack(pair)
 
     def extend_packset(self):
+        """ Follow dependency chains to find more candidates to put into
+            pairs.
+        """
         pack_count = self.packset.pack_count()
         while True:
             for pack in self.packset.packs:
@@ -375,11 +379,11 @@
 
     def combine_packset(self):
         """ Combination is done iterating the packs that have
-        a sorted op index of the first operation (= left).
-        If a pack is marked as 'full', the next pack that is
-        encountered having the full_pack.right == pack.left,
-        the pack is removed. This is because the packs have
-        intersecting edges.
+            a sorted op index of the first operation (= left).
+            If a pack is marked as 'full', the next pack that is
+            encountered having the full_pack.right == pack.left,
+            the pack is removed. This is because the packs have
+            intersecting edges.
         """
         if len(self.packset.packs) == 0:
             raise NotAVectorizeableLoop()
@@ -445,7 +449,6 @@
         for pack in self.packset.packs:
             pack.update_pack_of_nodes()
 
-
         if not we_are_translated():
             # some test cases check the accumulation variables
             self.packset.accum_vars = {}
@@ -470,6 +473,10 @@
                 assert False
 
     def schedule(self, vector=False, sched_data=None):
+        """ Scheduling the trace and emitting vector operations
+            for packed instructions.
+        """
+
         self.clear_newoperations()
         if sched_data is None:
             sched_data = VecScheduleData(self.cpu.vector_register_size,
@@ -520,7 +527,7 @@
 
     def prepend_invariant_operations(self, sched_data):
         """ Add invariant operations to the trace loop. returns the operation list
-            as first argument and a second a boolean value. it is true if any inva
+            as first argument and a second a boolean value.
         """
         oplist = self._newoperations
 
@@ -552,6 +559,11 @@
         return oplist
 
     def analyse_index_calculations(self):
+        """ Tries to move guarding instructions an all the instructions that
+            need to be computed for the guard to the loop header. This ensures
+            that guards fail 'early' and relax dependencies. Without this
+            step vectorization would not be possible!
+        """
         ee_pos = self.loop.find_first_index(rop.GUARD_EARLY_EXIT)
         if len(self.loop.operations) <= 2 or ee_pos == -1:
             raise NotAVectorizeableLoop()
@@ -627,6 +639,10 @@
 
 
 class CostModel(object):
+    """ Utility to estimate the savings for the new trace loop.
+        The main reaons to have this is of frequent unpack instructions,
+        and the missing ability (by design) to detect not vectorizable loops.
+    """
     def __init__(self, threshold, vec_reg_size):
         self.threshold = threshold
         self.vec_reg_size = vec_reg_size
@@ -657,7 +673,6 @@
         return self.savings >= 0
 
 class X86_CostModel(CostModel):
-
     def record_pack_savings(self, pack, times):
         cost, benefit_factor = (1,1)
         node = pack.operations[0]
@@ -693,7 +708,9 @@
         self.record_vector_pack(src, index, count)
 
 def isomorphic(l_op, r_op):
-    """ Subject of definition """
+    """ Subject of definition, here it is equal operation.
+        See limintations (vectorization.rst).
+    """
     if l_op.getopnum() == r_op.getopnum():
         return True
     return False
@@ -716,6 +733,8 @@
         self.packs.append(pack)
 
     def can_be_packed(self, lnode, rnode, origin_pack, forward):
+        """ Check to ensure that two nodes might be packed into a Pair.
+        """
         if isomorphic(lnode.getoperation(), rnode.getoperation()):
             if lnode.independent(rnode):
                 if forward and isinstance(origin_pack, AccumPair):
@@ -782,8 +801,9 @@
         return False
 
     def combine(self, i, j):
-        """ combine two packs. it is assumed that the attribute self.packs
-        is not iterated when calling this method. """
+        """ Combine two packs. it is assumed that the attribute self.packs
+            is not iterated when calling this method.
+        """
         pack_i = self.packs[i]
         pack_j = self.packs[j]
         operations = pack_i.operations
@@ -893,6 +913,4 @@
             # rename the variable with the box
             sched_data.setvector_of_box(accum.getoriginalbox(), 0, result) # prevent it from expansion
             renamer.start_renaming(accum.getoriginalbox(), result)
-            if not we_are_translated():
-                print "renaming accum", accum.getoriginalbox(), "->", result
 

From noreply at buildbot.pypy.org  Fri Aug 21 17:19:57 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 17:19:57 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: translation issues
Message-ID: <20150821151957.2FDB01C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79122:8cc48941c8b9
Date: 2015-08-21 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/8cc48941c8b9/

Log:	translation issues

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -196,6 +196,7 @@
         # this assumes that the root trace (= loop) is already compiled
         root = loop.versions[0]
         for faildescr in root.faildescrs:
+            assert isinstance(faildescr, CompileLoopVersionDescr)
             version = faildescr.version
             if not version or version.compiled():
                 continue
@@ -217,6 +218,7 @@
                 # to assign it's fail descr
                 continue
             for faildescr in lv.faildescrs:
+                assert isinstance(faildescr, CompileLoopVersionDescr)
                 version = faildescr.version
                 if version and version.compiled():
                     cpu.stitch_bridge(faildescr, version._compiled)
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -153,6 +153,7 @@
         operations[self.index] = None
         descr = self.op.getdescr()
         if descr and descr.loop_version():
+            assert isinstance(descr, CompileLoopVersionDescr)
             descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None

From noreply at buildbot.pypy.org  Fri Aug 21 18:52:14 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 18:52:14 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: reverted the repr_of_arg,
 added more debug info to update_accumlocation_loc
Message-ID: <20150821165214.2FFA41C0063@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79123:dc014b620f41
Date: 2015-08-21 18:52 +0200
http://bitbucket.org/pypy/pypy/changeset/dc014b620f41/

Log:	reverted the repr_of_arg, added more debug info to
	update_accumlocation_loc

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -337,14 +337,17 @@
         """
         assert isinstance(descr, ResumeGuardDescr)
         accum_info = descr.rd_accum_list
+        count = 0
         while accum_info:
             if accum_info.box is accum.getoriginalbox():
                 accum_info.loc = self.loc(arg)
                 accum_info.position = pos
                 break
+            count += 1
             accum_info = accum_info.prev
         else:
-            raise AssertionError("accum box has no accum_info entry")
+            llop.debug_print(lltype.Void, "[accumulator] %d accumulators, none matched box %s\n" % (count, accum_info.box))
+            assert 0
 
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):
         faillocs = self.locs_for_fail(guard_op)
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -102,7 +102,6 @@
         return descr.repr_of_descr()
 
     def repr_of_arg(self, arg):
-        return str(arg) # XXX
         try:
             mv = self.memo[arg]
         except KeyError:

From noreply at buildbot.pypy.org  Fri Aug 21 19:24:53 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 19:24:53 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: fixed issues with test cases for
	llgraph
Message-ID: <20150821172453.985F91C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79124:38a19b56ef23
Date: 2015-08-21 19:25 +0200
http://bitbucket.org/pypy/pypy/changeset/38a19b56ef23/

Log:	fixed issues with test cases for llgraph

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -17,6 +17,11 @@
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
 
+class LLAsmInfo(object):
+    def __init__(self, lltrace):
+        self.ops_offset = None
+        self.lltrace = lltrace
+
 class LLTrace(object):
     has_been_freed = False
     invalid = False
@@ -266,7 +271,7 @@
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
     def stitch_bridge(self, faildescr, target):
-        faildescr._llgraph_bridge = target
+        faildescr._llgraph_bridge = target.lltrace
 
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
                      unique_id=0, log=True, name='', logger=None):
@@ -285,7 +290,7 @@
         faildescr._llgraph_bridge = lltrace
         clt._llgraph_alltraces.append(lltrace)
         self._record_labels(lltrace)
-        return lltrace
+        return LLAsmInfo(lltrace)
 
     def _record_labels(self, lltrace):
         for i, op in enumerate(lltrace.operations):
@@ -753,6 +758,13 @@
             values.append(val)
         return values
 
+    def bh_vec_getarrayitem_gc(self, struct, offset, count, descr):
+        values = []
+        for i in range(count):
+            val = self.bh_getarrayitem_gc(struct, offset + i, descr)
+            values.append(val)
+        return values
+
     def bh_vec_raw_load(self, struct, offset, count, descr):
         values = []
         stride = descr.get_item_size_in_bytes()
@@ -770,6 +782,9 @@
         for i,n in enumerate(newvalues):
             self.bh_setarrayitem_raw(struct, offset + i, n, descr)
 
+    def bh_vec_setarrayitem_gc(self, struct, offset, newvalues, descr):
+        for i,n in enumerate(newvalues):
+            self.bh_setarrayitem_gc(struct, offset + i, n, descr)
 
     def store_fail_descr(self, deadframe, descr):
         pass # I *think*
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_guard.py b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_guard.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
@@ -319,12 +319,13 @@
         opt = self.optguards(loop1, True)
         self.assert_guard_count(loop1, 2)
         self.assert_contains_sequence(loop1, """
-        i40 = int_ge(42, i2)
-        guard_true(i40) []
         ...
         i10 = int_gt(i1, 42)
         guard_true(i10) []
         ...
+        i40 = int_gt(i11, i2)
+        guard_true(i40) []
+        ...
         """)
 
 class Test(GuardBaseTest, LLtypeMixin):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1006,7 +1006,7 @@
     asmlen - assembler block length
     rawstart - address a guard can jump to
     """
-    def __init__(self, ops_offset, asmaddr, asmlen, rawstart):
+    def __init__(self, ops_offset, asmaddr, asmlen, rawstart=0):
         self.ops_offset = ops_offset
         self.asmaddr = asmaddr
         self.asmlen = asmlen

From noreply at buildbot.pypy.org  Fri Aug 21 19:26:57 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Fri, 21 Aug 2015 19:26:57 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: updated command line vec parameter
Message-ID: <20150821172657.6525F1C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79125:754c30aac049
Date: 2015-08-21 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/754c30aac049/

Log:	updated command line vec parameter

diff --git a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
--- a/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_micronumpy.py
@@ -47,8 +47,8 @@
             return c.sum()
         """.format(op=op, adtype=adtype, bdtype=bdtype, count=count, a=a, b=b)
         exec py.code.Source(source).compile()
-        vlog = self.run(main, [], vectorize=1)
-        log = self.run(main, [], vectorize=0)
+        vlog = self.run(main, [], vec=1)
+        log = self.run(main, [], vec=0)
         assert log.result == vlog.result
         assert log.result == result
 

From noreply at buildbot.pypy.org  Fri Aug 21 20:11:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 21 Aug 2015 20:11:01 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: pass more tests, push push push
Message-ID: <20150821181101.37FB71C0063@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79126:a324ca2df937
Date: 2015-08-21 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/a324ca2df937/

Log:	pass more tests, push push push

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -500,28 +500,29 @@
         self.guards(info1, info2, InputArgRef(), value1, expected, [box])
 
     def test_generate_guards_on_virtual_fields_matches_instance(self):
-        innervalue1 = PtrOptValue(self.nodebox)
-        constclassbox = self.cpu.ts.cls_of_box(self.nodebox)
-        innervalue1.make_constant_class(None, constclassbox)
-        innerinfo1 = NotVirtualStateInfo(innervalue1)
+        classbox = self.cpu.ts.cls_of_box(InputArgRef(self.nodeaddr))
+        innervalue1 = info.InstancePtrInfo(classbox)
+        innerinfo1 = NotVirtualStateInfo(self.cpu, 'r', innervalue1)
         innerinfo1.position = 1
-        innerinfo2 = NotVirtualStateInfo(PtrOptValue(self.nodebox))
+        innerinfo2 = NotVirtualStateInfo(self.cpu, 'r', None)
         innerinfo2.position = 1
 
-        info1 = VirtualStateInfo(ConstInt(42), [1])
+        info1 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
         info1.fieldstate = [innerinfo1]
 
-        info2 = VirtualStateInfo(ConstInt(42), [1])
+        info2 = VirtualStateInfo(ConstInt(42), [self.nextdescr])
         info2.fieldstate = [innerinfo2]
 
-        value1 = VirtualValue(self.cpu, constclassbox, self.nodebox)
-        value1._fields = {1: PtrOptValue(self.nodebox)}
+        value1 = info.InstancePtrInfo(classbox, self.nodesize)
+        nodebox = InputArgRef(self.nodeaddr)
+        value1._fields = [None] * (self.nextdescr.get_index() + 1)
+        value1._fields[self.nextdescr.get_index()] = nodebox
 
         expected = """
         [p0]
         guard_nonnull_class(p0, ConstClass(node_vtable)) []
         """
-        self.guards(info1, info2, value1, expected, [self.nodebox])
+        self.guards(info1, info2, nodebox, value1, expected, [nodebox])
 
     def test_generate_guards_on_virtual_fields_matches_struct(self):
         innervalue1 = PtrOptValue(self.nodebox)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -119,9 +119,7 @@
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
         if box is not None:
-            yyy
-            assert isinstance(value, virtualize.AbstractVirtualStructValue)
-            assert value.is_virtual()
+            assert opinfo.is_virtual()
 
         if len(self.fielddescrs) != len(other.fielddescrs):
             raise VirtualStatesCantMatch("field descrs don't match")
@@ -132,8 +130,7 @@
             if box is not None:
                 fieldbox = opinfo._fields[self.fielddescrs[i].get_index()]
                 # must be there
-                xxx
-                fieldinfo = fieldbox.get_forwarded()
+                fieldinfo = get_forwarded(fieldbox)
             else:
                 fieldbox = None
                 fieldinfo = None

From noreply at buildbot.pypy.org  Fri Aug 21 21:11:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 21 Aug 2015 21:11:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a #define from recent CPython 2.x
Message-ID: <20150821191113.ACEDB1C0063@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79127:337ed0dae21f
Date: 2015-08-21 21:11 +0200
http://bitbucket.org/pypy/pypy/changeset/337ed0dae21f/

Log:	Add a #define from recent CPython 2.x

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -379,6 +379,8 @@
     PyObject *ht_name, *ht_slots;
 } PyHeapTypeObject;
 
+#define PyObject_Bytes PyObject_Str
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 

From noreply at buildbot.pypy.org  Sat Aug 22 01:22:29 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 22 Aug 2015 01:22:29 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: hg merge default
Message-ID: <20150821232229.E0BBF1C0063@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79128:61296bb0341b
Date: 2015-08-22 01:22 +0200
http://bitbucket.org/pypy/pypy/changeset/61296bb0341b/

Log:	hg merge default

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.0
+Version: 1.2.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.6.1.rst
    whatsnew-2.6.0.rst
    whatsnew-2.5.1.rst
    whatsnew-2.5.0.rst
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.6.1.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -1,6 +1,6 @@
-=======================
-What's new in PyPy 2.6+
-=======================
+========================
+What's new in PyPy 2.6.1
+========================
 
 .. this is a revision shortly after release-2.6.0
 .. startrev: 91904d5c5188
@@ -32,7 +32,10 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+Part of cffi 1.2.
+
 .. branch: cffi-new-allocator
+Part of cffi 1.2.
 
 .. branch: unicode-dtype
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,68 +2,6 @@
 What's new in PyPy 2.6+
 =======================
 
-.. this is a revision shortly after release-2.6.0
-.. startrev: 91904d5c5188
+.. this is a revision shortly after release-2.6.1
+.. startrev: 83ebc73d4fcb
 
-.. branch: use_min_scalar
-Correctly resolve the output dtype of ufunc(array, scalar) calls.
-
-.. branch: stdlib-2.7.10
-
-Update stdlib to version 2.7.10
-
-.. branch: issue2062
-
-.. branch: disable-unroll-for-short-loops
-The JIT no longer performs loop unrolling if the loop compiles to too much code.
-
-.. branch: run-create_cffi_imports
-
-Build cffi import libraries as part of translation by monkey-patching an 
-additional task into translation
-
-.. branch: int-float-list-strategy
-
-Use a compact strategy for Python lists that mix integers and floats,
-at least if the integers fit inside 32 bits.  These lists are now
-stored as an array of floats, like lists that contain only floats; the
-difference is that integers are stored as tagged NaNs.  (This should
-have no visible effect!  After ``lst = [42, 42.5]``, the value of
-``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
-
-.. branch: cffi-callback-onerror
-.. branch: cffi-new-allocator
-
-.. branch: unicode-dtype
-
-Partial implementation of unicode dtype and unicode scalars.
-
-.. branch: dtypes-compatability
-
-Improve compatibility with numpy dtypes; handle offsets to create unions,
-fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
-allow subclassing dtype
-
-.. branch: indexing
-
-Refactor array indexing to support ellipses.
-
-.. branch: numpy-docstrings
-
-Allow the docstrings of built-in numpy objects to be set at run-time.
-
-.. branch: nditer-revisited
-
-Implement nditer 'buffered' flag and fix some edge cases
-
-.. branch: ufunc-reduce
-
-Allow multiple axes in ufunc.reduce()
-
-.. branch: fix-tinylang-goals
-
-Update tinylang goals to match current rpython
-
-.. branch: vmprof-review
-
-Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.0"
+VERSION = "1.2.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,4 +3427,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,12 @@
             i = 0
             count = 0
             i += 5 * WORD # header
-            assert s[i] == 4
-            i += 1 # marker
-            assert s[i] == 4
-            i += 1 # length
-            i += len('pypy')
+            assert s[i    ] == 5    # MARKER_HEADER
+            assert s[i + 1] == 0    # 0
+            assert s[i + 2] == 1    # VERSION_THREAD_ID
+            assert s[i + 3] == 4    # len('pypy')
+            assert s[i + 4: i + 8] == 'pypy'
+            i += 8
             while i < len(s):
                 if s[i] == 3:
                     break
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -42,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
+""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -364,6 +364,8 @@
     PyObject *ht_name, *ht_slots;
 } PyHeapTypeObject;
 
+#define PyObject_Bytes PyObject_Str
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -160,6 +160,35 @@
     assert func.name == 'sin'
     assert func.BType == ', ), , False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -238,6 +267,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,6 +25,9 @@
     if 1:     # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
+    else:
+        kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+                                      ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -318,15 +318,32 @@
                 import cffi
                 ffi = cffi.FFI()
                 ffi.set_source("pack3.mymod", "/*code would be here*/")
+                ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
+                from distutils.command.build_ext import build_ext
+                import os
+
+                class TestBuildExt(build_ext):
+                    def pre_run(self, ext, ffi):
+                        print('_make_setuptools_api: in pre_run:', end=" ")
+                        assert ffi._hi_there == 42
+                        assert ext.name == "pack3.mymod"
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print('creating %r' % (fn,))
+                        open(fn, 'w').close()
+
                 setup(name='example1',
                       version='0.1',
                       packages=['pack3'],
                       package_dir={'': 'src1'},
-                      cffi_modules=["src1/pack3/_build.py:ffi"])
+                      cffi_modules=["src1/pack3/_build.py:ffi"],
+                      cmdclass={'build_ext': TestBuildExt},
+                      )
             """)
 
     @chdir_to_tmp
@@ -335,6 +352,7 @@
         self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None}}})
 
@@ -344,6 +362,7 @@
         self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None,
                                                       'mymod.SO': None}}})
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -13,6 +13,11 @@
 
 def _assert_rpythonic(func):
     """Raise ValueError if ``func`` is obviously not RPython"""
+    try:
+        func.func_code.co_cellvars
+    except AttributeError:
+        raise ValueError("%r is not RPython: it is likely an unexpected "
+                         "built-in function or type" % (func,))
     if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
         raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
     if func.func_code.co_cellvars:
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1363,6 +1363,15 @@
         simplify_graph(graph)
         assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
 
+    def test_unexpected_builtin_function(self):
+        import itertools
+        e = py.test.raises(ValueError, build_flow, itertools.permutations)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, itertools.tee)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, Exception.__init__)
+        assert ' is not RPython:' in str(e.value)
+
 
 DATA = {'x': 5,
         'y': 6}
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -64,12 +64,6 @@
                                               operations,
                                               original_loop_token, log=log)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU_ARM.cast_adr_to_int(adr)
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -636,8 +636,7 @@
             assert isinstance(box, Box)
             loc = self.fm.get_new_loc(box)
             locs.append(loc.value - base_ofs)
-        if looptoken.compiled_loop_token is not None:
-            # for tests
+        if looptoken.compiled_loop_token is not None:   # <- for tests
             looptoken.compiled_loop_token._ll_initial_locs = locs
 
     def can_merge_with_next_guard(self, op, i, operations):
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -6,9 +6,9 @@
 #endif
 
 #ifdef RPYTHON_VMPROF
-RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored);
+RPY_EXTERN void vmprof_ignore_signals(int ignored);
 static void pypy_codemap_invalid_set(int ignored) {
-    rpython_vmprof_ignore_signals(ignored);
+    vmprof_ignore_signals(ignored);
 }
 #else
 static void pypy_codemap_invalid_set(int ignored) {
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -100,12 +100,6 @@
         return self.assembler.assemble_bridge(faildescr, inputargs, operations,
                                               original_loop_token, log, logger)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU386.cast_adr_to_int(adr)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -268,6 +268,7 @@
                 y -= 1
             return res
         res = self.meta_interp(f, [6, sys.maxint, 48])
+        self.check_trace_count(6)
         assert res == f(6, sys.maxint, 48)
 
     def test_loop_invariant_mul_bridge_ovf2(self):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,6 +1087,16 @@
     """
     assert type(value) is cls
 
+def ll_record_exact_class(ll_value, ll_cls):
+    from rpython.rlib.debug import ll_assert
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.rclass import ll_type
+    ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument")
+    ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments")
+    llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls)
+
+
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_class
 
@@ -1100,12 +1110,10 @@
         from rpython.rtyper import rclass
 
         classrepr = rclass.get_type_repr(hop.rtyper)
-
-        hop.exception_cannot_occur()
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
         v_cls = hop.inputarg(classrepr, arg=1)
-        return hop.genop('jit_record_exact_class', [v_inst, v_cls],
-                         resulttype=lltype.Void)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
     pass
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,24 +40,20 @@
         **eci_kwds))
 
 
-    vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP,
-                                  compilation_info=eci)
-    vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT,
+    vmprof_init = rffi.llexternal("vmprof_init",
+                                  [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+                                  rffi.CCHARP, compilation_info=eci)
+    vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT,
+    vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
                                      compilation_info=eci,
                                      save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf",
-                                       [rffi.CCHARP, rffi.LONG],
-                                       lltype.Void, compilation_info=eci)
-
-    ## vmprof_register_virtual_function = rffi.llexternal(
-    ##     "vmprof_register_virtual_function",
-    ##     [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    ##     compilation_info=eci, _nowrapper=True)
-
-    vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals",
+    vmprof_register_virtual_function = rffi.llexternal(
+                                           "vmprof_register_virtual_function",
+                                           [rffi.CCHARP, rffi.LONG, rffi.INT],
+                                           rffi.INT, compilation_info=eci)
+    vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals",
                                             [rffi.INT], lltype.Void,
                                             compilation_info=eci)
     return CInterface(locals())
@@ -83,10 +79,20 @@
 
     cont_name = 'rpyvmprof_f_%s_%s' % (name, token)
     tramp_name = 'rpyvmprof_t_%s_%s' % (name, token)
+    orig_tramp_name = tramp_name
 
     func.c_name = cont_name
     func._dont_inline_ = True
 
+    if sys.platform == 'darwin':
+        # according to internet "At the time UNIX was written in 1974...."
+        # "... all C functions are prefixed with _"
+        cont_name = '_' + cont_name
+        tramp_name = '_' + tramp_name
+        PLT = ""
+    else:
+        PLT = "@PLT"
+
     assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
         "rvmprof only supports x86-64 CPUs for now")
 
@@ -111,17 +117,15 @@
     target.write("""\
 \t.text
 \t.globl\t%(tramp_name)s
-\t.type\t%(tramp_name)s, @function
 %(tramp_name)s:
 \t.cfi_startproc
 \tpushq\t%(reg)s
 \t.cfi_def_cfa_offset 16
-\tcall %(cont_name)s at PLT
+\tcall %(cont_name)s%(PLT)s
 \taddq\t$8, %%rsp
 \t.cfi_def_cfa_offset 8
 \tret
 \t.cfi_endproc
-\t.size\t%(tramp_name)s, .-%(tramp_name)s
 """ % locals())
 
     def tok2cname(tok):
@@ -133,7 +137,7 @@
 
     header = 'RPY_EXTERN %s %s(%s);\n' % (
         tok2cname(restok),
-        tramp_name,
+        orig_tramp_name,
         ', '.join([tok2cname(tok) for tok in token] + ['long']))
 
     header += """\
@@ -147,7 +151,7 @@
 #endif
 #define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s
 }
-""" % (tramp_name, tramp_name, tramp_name)
+""" % (tramp_name, orig_tramp_name, tramp_name)
 
     eci = ExternalCompilationInfo(
         post_include_bits = [header],
@@ -155,7 +159,7 @@
     )
 
     return rffi.llexternal(
-        tramp_name,
+        orig_tramp_name,
         [token2lltype(tok) for tok in token] + [lltype.Signed],
         token2lltype(restok),
         compilation_info=eci,
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,14 +1,12 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import jit, rgc, rposix
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
 from rpython.rtyper.lltypesystem import rffi
 
-MAX_CODES = 8000 - 255
-MAX_FUNC_NAME = 255
+MAX_FUNC_NAME = 1023
 
 # ____________________________________________________________
 
@@ -34,8 +32,6 @@
 
     def _cleanup_(self):
         self.is_enabled = False
-        self.fileno = -1
-        self._current_codes = None
 
     @specialize.argtype(1)
     def register_code(self, code, full_name_func):
@@ -102,18 +98,13 @@
         assert fileno >= 0
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
-        if not (1e-6 <= interval < 1.0):
-            raise VMProfError("bad value for 'interval'")
-        interval_usec = int(interval * 1000000.0)
 
-        p_error = self.cintf.vmprof_init(fileno)
+        p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
-        self.fileno = fileno
-        self._write_header(interval_usec)
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable(interval_usec)
+        res = self.cintf.vmprof_enable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
@@ -125,9 +116,6 @@
         if not self.is_enabled:
             raise VMProfError("vmprof is not enabled")
         self.is_enabled = False
-        if self._current_codes is not None:
-            self._flush_codes()
-        self.fileno = -1
         res = self.cintf.vmprof_disable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
@@ -136,48 +124,8 @@
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
             "and at most %d characters; got '%s'" % (MAX_FUNC_NAME, name))
-        b = self._current_codes
-        if b is None:
-            b = self._current_codes = StringBuilder()
-        b.append('\x02')
-        _write_long_to_string_builder(uid, b)
-        _write_long_to_string_builder(len(name), b)
-        b.append(name)
-        if b.getlength() >= MAX_CODES:
-            self._flush_codes()
-
-    def _flush_codes(self):
-        buf = self._current_codes.build()
-        self._current_codes = None
-        self.cintf.vmprof_write_buf(buf, len(buf))
-        # NOTE: keep in mind that vmprof_write_buf() can only write
-        # a maximum of 8184 bytes.  This should be guaranteed here because:
-        assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184
-
-    def _write_header(self, interval_usec):
-        b = StringBuilder()
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(3, b)
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(interval_usec, b)
-        _write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
-        buf = b.build()
-        self.cintf.vmprof_write_buf(buf, len(buf))
-
-
-def _write_long_to_string_builder(l, b):
-    b.append(chr(l & 0xff))
-    b.append(chr((l >> 8) & 0xff))
-    b.append(chr((l >> 16) & 0xff))
-    b.append(chr((l >> 24) & 0xff))
-    if sys.maxint > 2147483647:
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
+        if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
+            raise VMProfError("vmprof buffers full!  disk full or too slow")
 
 
 def vmprof_execute_code(name, get_code_fn, result_class=None):
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -1,22 +1,3 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in
- * gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *   Armin Rigo - arigo at tunes.org
- *
- */
-
 #define _GNU_SOURCE 1
 
 
@@ -39,431 +20,4 @@
 #endif
 
 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
-
-
-/************************************************************/
-
-// functions copied from libunwind using dlopen
-
-static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-static int (*unw_step)(unw_cursor_t*) = NULL;
-static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static int profile_file = -1;
-
-
-RPY_EXTERN
-char *rpython_vmprof_init(int fd)
-{
-    if (!unw_get_reg) {
-        void *libhandle;
-
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
-            goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
-            goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
-            goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
-            goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
-            goto error;
-    }
-    if (prepare_concurrent_bufs() < 0)
-        return "out of memory";
-
-    assert(fd >= 0);
-    profile_file = fd;
-    return NULL;
-
- error:
-    return dlerror();
-}
-
-/************************************************************/
-
-/* value: last bit is 1 if signals must be ignored; all other bits
-   are a counter for how many threads are currently in a signal handler */
-static long volatile signal_handler_value = 1;
-
-RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
-{
-    if (!ignored) {
-        __sync_fetch_and_and(&signal_handler_value, ~1L);
-    }
-    else {
-        /* set the last bit, and wait until concurrently-running signal
-           handlers finish */
-        while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
-            usleep(1);
-        }
-    }
-}
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH   \
-    ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-struct prof_stacktrace_s {
-    char padding[sizeof(long) - 1];
-    char marker;
-    long count, depth;
-    void *stack[];
-};
-
-static long profile_interval_usec = 0;
-static char atfork_hook_installed = 0;
-
-
-/* ******************************************************
- * libunwind workaround for process JIT frames correctly
- * ******************************************************
- */
-
-#include "rvmprof_get_custom_offset.h"
-
-typedef struct {
-    void* _unused1;
-    void* _unused2;
-    void* sp;
-    void* ip;
-    void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4];
-} vmprof_hacked_unw_cursor_t;
-
-static int vmprof_unw_step(unw_cursor_t *cp, int first_run)
-{
-    void* ip;
-    void* sp;
-    ptrdiff_t sp_offset;
-    unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip);
-    unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
-    if (!first_run) {
-        // make sure we're pointing to the CALL and not to the first
-        // instruction after. If the callee adjusts the stack for us
-        // it's not safe to be at the instruction after
-        ip -= 1;
-    }
-    sp_offset = vmprof_unw_get_custom_offset(ip, cp);
-
-    if (sp_offset == -1) {
-        // it means that the ip is NOT in JITted code, so we can use the
-        // stardard unw_step
-        return unw_step(cp);
-    }
-    else {
-        // this is a horrible hack to manually walk the stack frame, by
-        // setting the IP and SP in the cursor
-        vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp;
-        void* bp = (void*)sp + sp_offset;
-        cp2->sp = bp;
-        bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext)
-{
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    (void)ret;
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) {
-            // found main loop stack frame
-            void* sp;
-            unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-            void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
-            void **arg_ptr = (void**)arg_addr;
-            /* if (mainloop_get_virtual_ip) {
-               ip = mainloop_get_virtual_ip(*arg_ptr);
-               } else { */
-            ip = *arg_ptr;
-        }
-
-        int first_run = (n == 0);
-        result[n++] = ip;
-        n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0)
-            break;
-    }
-    return n;
-}
-
-
-/* *************************************************************
- * the signal handler
- * *************************************************************
- */
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
-{
-    long val = __sync_fetch_and_add(&signal_handler_value, 2L);
-
-    if ((val & 1) == 0) {
-        int saved_errno = errno;
-        int fd = profile_file;
-        assert(fd >= 0);
-
-        struct profbuf_s *p = reserve_buffer(fd);
-        if (p == NULL) {
-            /* ignore this signal: there are no free buffers right now */
-        }
-        else {
-            int depth;
-            struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
-            st->marker = MARKER_STACKTRACE;
-            st->count = 1;
-            st->stack[0] = GetPC((ucontext_t*)ucontext);
-            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
-            depth++;  // To account for pc value in stack[0];
-            st->depth = depth;
-            p->data_offset = offsetof(struct prof_stacktrace_s, marker);
-            p->data_size = (depth * sizeof(void *) +
-                            sizeof(struct prof_stacktrace_s) -
-                            offsetof(struct prof_stacktrace_s, marker));
-            commit_buffer(fd, p);
-        }
-
-        errno = saved_errno;
-    }
-
-    __sync_sub_and_fetch(&signal_handler_value, 2L);
-}
-
-
-/* *************************************************************
- * the setup and teardown functions
- * *************************************************************
- */
-
-static int install_sigprof_handler(void)
-{
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-        sigaction(SIGPROF, &sa, NULL) == -1)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_handler(void)
-{
-    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
-        return -1;
-    return 0;
-}
-
-static int install_sigprof_timer(void)
-{
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = profile_interval_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (profile_interval_usec > 0) {
-        remove_sigprof_timer();
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (profile_interval_usec > 0) {
-        install_sigprof_timer();
-    }
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
-{
-    assert(profile_file >= 0);
-    assert(interval_usec > 0);
-    profile_interval_usec = interval_usec;
-
-    if (install_pthread_atfork_hooks() == -1)
-        goto error;
-    if (install_sigprof_handler() == -1)
-        goto error;
-    if (install_sigprof_timer() == -1)
-        goto error;
-    rpython_vmprof_ignore_signals(0);
-    return 0;
-
- error:
-    profile_file = -1;
-    profile_interval_usec = 0;
-    return -1;
-}
-
-static int _write_all(const void *buf, size_t bufsize)
-{
-    while (bufsize > 0) {
-        ssize_t count = write(profile_file, buf, bufsize);
-        if (count <= 0)
-            return -1;   /* failed */
-        buf += count;
-        bufsize -= count;
-    }
-    return 0;
-}
-
-static int close_profile(void)
-{
-    char buf[4096];
-    ssize_t size;
-    unsigned char marker = MARKER_TRAILER;
-
-    if (_write_all(&marker, 1) < 0)
-        return -1;
-
-#ifdef __linux__
-    // copy /proc/self/maps to the end of the profile file
-    int srcfd = open("/proc/self/maps", O_RDONLY);
-    if (srcfd < 0)
-        return -1;
-
-    while ((size = read(srcfd, buf, sizeof buf)) > 0) {
-        if (_write_all(buf, size) < 0) {
-            close(srcfd);
-            return -1;
-        }
-    }
-    close(srcfd);
-#else
-    // freebsd and mac
-    sprintf(buf, "procstat -v %d", getpid());
-    FILE *srcf = popen(buf, "r");
-    if (!srcf)
-        return -1;
-
-    while ((size = fread(buf, 1, sizeof buf, src))) {
-        if (_write_all(buf, size) < 0) {
-            pclose(srcf);
-            return -1;
-        }
-    }
-    pclose(srcf);
-#endif
-
-    /* don't close() the file descriptor from here */
-    profile_file = -1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_disable(void)
-{
-    rpython_vmprof_ignore_signals(1);
-    profile_interval_usec = 0;
-
-    if (remove_sigprof_timer() == -1)
-        return -1;
-    if (remove_sigprof_handler() == -1)
-        return -1;
-    if (shutdown_concurrent_bufs(profile_file) < 0)
-        return -1;
-    return close_profile();
-}
-
-RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
-{
-    struct profbuf_s *p;
-
-    while ((p = reserve_buffer(profile_file)) == NULL) {
-        /* spin loop waiting for a buffer to be ready; should almost never
-           be the case */
-        usleep(1);
-    }
-
-    if (size > SINGLE_BUF_SIZE)
-        size = SINGLE_BUF_SIZE;
-    memcpy(p->data, buf, size);
-    p->data_size = size;
-
-    commit_buffer(profile_file, p);
-}
+#include "vmprof_main.h"
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
 
-RPY_EXTERN char *rpython_vmprof_init(int);
-RPY_EXTERN void rpython_vmprof_ignore_signals(int);
-RPY_EXTERN int rpython_vmprof_enable(long);
-RPY_EXTERN int rpython_vmprof_disable(void);
-RPY_EXTERN void rpython_vmprof_write_buf(char *, long);
+RPY_EXTERN char *vmprof_init(int, double, char *);
+RPY_EXTERN void vmprof_ignore_signals(int);
+RPY_EXTERN int vmprof_enable(void);
+RPY_EXTERN int vmprof_disable(void);
+RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
rename from rpython/rlib/rvmprof/src/rvmprof_config.h
rename to rpython/rlib/rvmprof/src/vmprof_config.h
diff --git a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
rename from rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
rename to rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
--- a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -7,12 +7,69 @@
 #endif
 
 
+#ifdef CPYTHON_GET_CUSTOM_OFFSET
+static void *tramp_start, *tramp_end;
+#endif
+
+
 static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
+
+#if defined(PYPY_JIT_CODEMAP)
+
     intptr_t ip_l = (intptr_t)ip;
     return pypy_jit_stack_depth_at_loc(ip_l);
+
+#elif defined(CPYTHON_GET_CUSTOM_OFFSET)
+
+    if (ip >= tramp_start && ip <= tramp_end) {
+        // XXX the return value is wrong for all the places before push and
+        //     after pop, fix
+        void *bp;
+        void *sp;
+
+        /* This is a stage2 trampoline created by hotpatch:
+
+               push   %rbx
+               push   %rbp
+               mov    %rsp,%rbp
+               and    $0xfffffffffffffff0,%rsp   // make sure the stack is aligned
+               movabs $0x7ffff687bb10,%rbx
+               callq  *%rbx
+               leaveq 
+               pop    %rbx
+               retq   
+
+           the stack layout is like this:
+
+               +-----------+                      high addresses
+               | ret addr  |
+               +-----------+
+               | saved rbx |   start of the function frame
+               +-----------+
+               | saved rbp |
+               +-----------+
+               | ........  |   <-- rbp
+               +-----------+                      low addresses
+
+           So, the trampoline frame starts at rbp+16, and the return address,
+           is at rbp+24.  The vmprof API requires us to return the offset of
+           the frame relative to sp, hence we have this weird computation.
+
+           XXX (antocuni): I think we could change the API to return directly
+           the frame address instead of the offset; however, this require a
+           change in the PyPy code too
+        */
+
+        unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
+        unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp);
+        return bp+16+8-sp;
+    }
+    return -1;
+
 #else
+
     return -1;
+
 #endif
 }
 
diff --git a/rpython/rlib/rvmprof/src/rvmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
rename from rpython/rlib/rvmprof/src/rvmprof_getpc.h
rename to rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/rvmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -44,7 +44,7 @@
 #ifndef BASE_GETPC_H_
 #define BASE_GETPC_H_
 
-#include "rvmprof_config.h"
+#include "vmprof_config.h"
 
 // On many linux systems, we may need _GNU_SOURCE to get access to
 // the defined constants that define the register we want to see (eg
@@ -53,7 +53,9 @@
 // If #define _GNU_SOURCE causes problems, this might work instead.
 // It will cause problems for FreeBSD though!, because it turns off
 // the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
+#ifdef __APPLE__
+#define _XOPEN_SOURCE 500
+#endif
 
 #include          // for memcmp
 #if defined(HAVE_SYS_UCONTEXT_H)
@@ -179,7 +181,11 @@
 // configure.ac (or set it manually in your config.h).
 #else
 void* GetPC(ucontext_t *signal_ucontext) {
+#ifdef __APPLE__
+  return (void*)(signal_ucontext->uc_mcontext->__ss.__rip);
+#else
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
+#endif
 }
 
 #endif
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/vmprof_main.h
copy from rpython/rlib/rvmprof/src/rvmprof.c
copy to rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -19,26 +19,6 @@
 
 #define _GNU_SOURCE 1
 
-
-#ifdef RPYTHON_LL2CTYPES
-   /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */
-#  ifndef RPY_EXTERN
-#    define RPY_EXTERN RPY_EXPORTED
-#  endif
-#  define RPY_EXPORTED  extern __attribute__((visibility("default")))
-#  define VMPROF_ADDR_OF_TRAMPOLINE(addr)  0
-
-#else
-
-#  include "common_header.h"
-#  include "rvmprof.h"
-#  ifndef VMPROF_ADDR_OF_TRAMPOLINE
-#   error "RPython program using rvmprof, but not calling vmprof_execute_code()"
-#  endif
-
-#endif
-
-
 #include 
 #include 
 #include 
@@ -47,11 +27,12 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
+#include "vmprof_getpc.h"
+#include "vmprof_unwind.h"
+#include "vmprof_mt.h"
 
 
 /************************************************************/
@@ -64,23 +45,40 @@
 static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
 
 static int profile_file = -1;
+static long prepare_interval_usec;
+static struct profbuf_s *volatile current_codes;
+static void *(*mainloop_get_virtual_ip)(char *) = 0;
 
+static int opened_profile(char *interp_name);
+static void flush_codes(void);
+
+#ifdef __APPLE__
+#define UNWIND_NAME "/usr/lib/system/libunwind.dylib"
+#define UNW_PREFIX "unw"
+#else
+#define UNWIND_NAME "libunwind.so"
+#define UNW_PREFIX "_ULx86_64"
+#endif
 
 RPY_EXTERN
-char *rpython_vmprof_init(int fd)
+char *vmprof_init(int fd, double interval, char *interp_name)
 {
+    if (interval < 1e-6 || interval >= 1.0)
+        return "bad value for 'interval'";
+    prepare_interval_usec = (int)(interval * 1000000.0);
+
     if (!unw_get_reg) {
         void *libhandle;
 
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
+        if (!(libhandle = dlopen(UNWIND_NAME, RTLD_LAZY | RTLD_LOCAL)))
             goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
+        if (!(unw_get_reg = dlsym(libhandle, UNW_PREFIX "_get_reg")))
             goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
+        if (!(unw_get_proc_info = dlsym(libhandle, UNW_PREFIX "_get_proc_info")))
             goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
+        if (!(unw_init_local = dlsym(libhandle, UNW_PREFIX  "_init_local")))
             goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
+        if (!(unw_step = dlsym(libhandle, UNW_PREFIX  "_step")))
             goto error;
     }
     if (prepare_concurrent_bufs() < 0)
@@ -88,6 +86,10 @@
 
     assert(fd >= 0);
     profile_file = fd;
+    if (opened_profile(interp_name) < 0) {
+        profile_file = -1;
+        return strerror(errno);
+    }
     return NULL;
 
  error:
@@ -101,7 +103,7 @@
 static long volatile signal_handler_value = 1;
 
 RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
+void vmprof_ignore_signals(int ignored)
 {
     if (!ignored) {
         __sync_fetch_and_and(&signal_handler_value, ~1L);
@@ -128,6 +130,11 @@
 #define MARKER_STACKTRACE '\x01'
 #define MARKER_VIRTUAL_IP '\x02'
 #define MARKER_TRAILER '\x03'
+#define MARKER_INTERP_NAME '\x04'   /* deprecated */
+#define MARKER_HEADER '\x05'
+
+#define VERSION_BASE '\x00'
+#define VERSION_THREAD_ID '\x01'
 
 struct prof_stacktrace_s {
     char padding[sizeof(long) - 1];
@@ -145,7 +152,7 @@
  * ******************************************************
  */
 
-#include "rvmprof_get_custom_offset.h"
+#include "vmprof_get_custom_offset.h"
 
 typedef struct {
     void* _unused1;
@@ -220,12 +227,10 @@
             // found main loop stack frame
             void* sp;
             unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-            void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
-            void **arg_ptr = (void**)arg_addr;
-            /* if (mainloop_get_virtual_ip) {
-               ip = mainloop_get_virtual_ip(*arg_ptr);
-               } else { */
-            ip = *arg_ptr;
+            if (mainloop_get_virtual_ip)
+                ip = mainloop_get_virtual_ip((char *)sp);
+            else
+                ip = *(void **)sp;
         }
 
         int first_run = (n == 0);
@@ -237,6 +242,23 @@
     return n;
 }
 
+static void *get_current_thread_id(void)
+{
+    /* xxx This function is a hack on two fronts:
+
+       - It assumes that pthread_self() is async-signal-safe.  This
+         should be true on Linux.  I hope it is also true elsewhere.
+
+       - It abuses pthread_self() by assuming it just returns an
+         integer.  According to comments in CPython's source code, the
+         platforms where it is not the case are rare nowadays.
+
+       An alternative would be to try to look if the information is
+       available in the ucontext_t in the caller.
+    */
+    return (void *)pthread_self();
+}
+
 
 /* *************************************************************
  * the signal handler
@@ -262,9 +284,10 @@
             st->marker = MARKER_STACKTRACE;
             st->count = 1;
             st->stack[0] = GetPC((ucontext_t*)ucontext);
-            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
+            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-2, ucontext);
             depth++;  // To account for pc value in stack[0];
             st->depth = depth;
+            st->stack[depth++] = get_current_thread_id();
             p->data_offset = offsetof(struct prof_stacktrace_s, marker);
             p->data_size = (depth * sizeof(void *) +
                             sizeof(struct prof_stacktrace_s) -
@@ -358,11 +381,11 @@
 }
 
 RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
+int vmprof_enable(void)
 {
     assert(profile_file >= 0);
-    assert(interval_usec > 0);
-    profile_interval_usec = interval_usec;
+    assert(prepare_interval_usec > 0);
+    profile_interval_usec = prepare_interval_usec;
 
     if (install_pthread_atfork_hooks() == -1)
         goto error;
@@ -370,7 +393,7 @@
         goto error;
     if (install_sigprof_timer() == -1)
         goto error;
-    rpython_vmprof_ignore_signals(0);
+    vmprof_ignore_signals(0);
     return 0;
 
  error:
@@ -391,6 +414,29 @@
     return 0;
 }
 
+static int opened_profile(char *interp_name)
+{
+    struct {
+        long hdr[5];
+        char interp_name[259];
+    } header;
+
+    size_t namelen = strnlen(interp_name, 255);
+    current_codes = NULL;
+
+    header.hdr[0] = 0;
+    header.hdr[1] = 3;
+    header.hdr[2] = 0;
+    header.hdr[3] = prepare_interval_usec;
+    header.hdr[4] = 0;
+    header.interp_name[0] = MARKER_HEADER;
+    header.interp_name[1] = '\x00';
+    header.interp_name[2] = VERSION_THREAD_ID;
+    header.interp_name[3] = namelen;
+    memcpy(&header.interp_name[4], interp_name, namelen);
+    return _write_all(&header, 5 * sizeof(long) + 4 + namelen);
+}
+
 static int close_profile(void)
 {
     char buf[4096];
@@ -415,12 +461,16 @@
     close(srcfd);
 #else
     // freebsd and mac
+#if defined(__APPLE__)
+	sprintf(buf, "vmmap %d", getpid());
+#else
     sprintf(buf, "procstat -v %d", getpid());
+#endif
     FILE *srcf = popen(buf, "r");
     if (!srcf)
         return -1;
 
-    while ((size = fread(buf, 1, sizeof buf, src))) {
+    while ((size = fread(buf, 1, sizeof buf, srcf))) {
         if (_write_all(buf, size) < 0) {
             pclose(srcf);
             return -1;
@@ -435,35 +485,85 @@
 }
 
 RPY_EXTERN
-int rpython_vmprof_disable(void)
+int vmprof_disable(void)
 {
-    rpython_vmprof_ignore_signals(1);
+    vmprof_ignore_signals(1);
     profile_interval_usec = 0;
 
     if (remove_sigprof_timer() == -1)
         return -1;
     if (remove_sigprof_handler() == -1)
         return -1;
+    flush_codes();
     if (shutdown_concurrent_bufs(profile_file) < 0)
         return -1;
     return close_profile();
 }
 
 RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
+int vmprof_register_virtual_function(char *code_name, long code_uid,
+                                     int auto_retry)
 {
+    long namelen = strnlen(code_name, 1023);
+    long blocklen = 1 + 2 * sizeof(long) + namelen;
     struct profbuf_s *p;
+    char *t;
 
-    while ((p = reserve_buffer(profile_file)) == NULL) {
-        /* spin loop waiting for a buffer to be ready; should almost never
-           be the case */
-        usleep(1);
+ retry:
+    p = current_codes;
+    if (p != NULL) {
+        if (__sync_bool_compare_and_swap(¤t_codes, p, NULL)) {
+            /* grabbed 'current_codes': we will append the current block
+               to it if it contains enough room */
+            size_t freesize = SINGLE_BUF_SIZE - p->data_size;
+            if (freesize < blocklen) {
+                /* full: flush it */
+                commit_buffer(profile_file, p);
+                p = NULL;
+            }
+        }
+        else {
+            /* compare-and-swap failed, don't try again */
+            p = NULL;
+        }
     }
 
-    if (size > SINGLE_BUF_SIZE)
-        size = SINGLE_BUF_SIZE;
-    memcpy(p->data, buf, size);
-    p->data_size = size;
+    if (p == NULL) {
+        p = reserve_buffer(profile_file);
+        if (p == NULL) {
+            /* can't get a free block; should almost never be the
+               case.  Spin loop if allowed, or return a failure code
+               if not (e.g. we're in a signal handler) */
+            if (auto_retry > 0) {
+                auto_retry--;
+                usleep(1);
+                goto retry;
+            }
+            return -1;
+        }
+    }
 
-    commit_buffer(profile_file, p);
+    t = p->data + p->data_size;
+    p->data_size += blocklen;
+    assert(p->data_size <= SINGLE_BUF_SIZE);
+    *t++ = MARKER_VIRTUAL_IP;
+    memcpy(t, &code_uid, sizeof(long)); t += sizeof(long);
+    memcpy(t, &namelen, sizeof(long)); t += sizeof(long);
+    memcpy(t, code_name, namelen);
+
+    /* try to reattach 'p' to 'current_codes' */
+    if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) {
+        /* failed, flush it */
+        commit_buffer(profile_file, p);
+    }
+    return 0;
 }
+
+static void flush_codes(void)
+{
+    struct profbuf_s *p = current_codes;
+    if (p != NULL) {
+        current_codes = NULL;
+        commit_buffer(profile_file, p);
+    }
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
rename from rpython/rlib/rvmprof/src/rvmprof_mt.h
rename to rpython/rlib/rvmprof/src/vmprof_mt.h
--- a/rpython/rlib/rvmprof/src/rvmprof_mt.h
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -66,7 +66,7 @@
     unprepare_concurrent_bufs();
     profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
                                PROT_READ | PROT_WRITE,
-                               MAP_PRIVATE | MAP_ANONYMOUS,
+                               MAP_PRIVATE | MAP_ANON,
                                -1, 0);
     if (profbuf_all_buffers == MAP_FAILED) {
         profbuf_all_buffers = NULL;
@@ -190,10 +190,17 @@
     }
 }
 
+static void cancel_buffer(struct profbuf_s *buf)
+{
+    long i = buf - profbuf_all_buffers;
+    assert(profbuf_state[i] == PROFBUF_FILLING);
+    profbuf_state[i] = PROFBUF_UNUSED;
+}
+
 static int shutdown_concurrent_bufs(int fd)
 {
     /* no signal handler can be running concurrently here, because we
-       already did rpython_vmprof_ignore_signals(1) */
+       already did vmprof_ignore_signals(1) */
     assert(profbuf_write_lock == 0);
     profbuf_write_lock = 2;
 
diff --git a/rpython/rlib/rvmprof/src/rvmprof_unwind.h b/rpython/rlib/rvmprof/src/vmprof_unwind.h
rename from rpython/rlib/rvmprof/src/rvmprof_unwind.h
rename to rpython/rlib/rvmprof/src/vmprof_unwind.h
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,4 +1,7 @@
-import time, os, sys, py
+import time, os, sys
+if __name__ == '__main__':
+    sys.path += ['../../../..']    # for subprocess in test_interpreted
+import py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
 from rpython.translator.c.test.test_genc import compile
@@ -57,7 +60,8 @@
 def test_interpreted():
     # takes forever if the Python process is already big...
     import subprocess
-    subprocess.check_call([sys.executable, __file__])
+    subprocess.check_call([sys.executable, os.path.basename(__file__)],
+                          cwd=(os.path.dirname(__file__) or '.'))
 
 def test_compiled():
     fn = compile(main, [], gcpolicy="minimark")
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -616,26 +616,33 @@
                 while rbase.classdef is not None:
                     immutable_fields.update(rbase.immutable_field_set)
                     rbase = rbase.rbase
-                self._parse_field_list(immutable_fields, accessor)
+                self._parse_field_list(immutable_fields, accessor, hints)
 
-    def _parse_field_list(self, fields, accessor):
+    def _parse_field_list(self, fields, accessor, hints):
         ranking = {}
         for name in fields:
+            quasi = False
             if name.endswith('?[*]'):   # a quasi-immutable field pointing to
                 name = name[:-4]        # an immutable array
                 rank = IR_QUASIIMMUTABLE_ARRAY
+                quasi = True
             elif name.endswith('[*]'):    # for virtualizables' lists
                 name = name[:-3]
                 rank = IR_IMMUTABLE_ARRAY
             elif name.endswith('?'):    # a quasi-immutable field
                 name = name[:-1]
                 rank = IR_QUASIIMMUTABLE
+                quasi = True
             else:                       # a regular immutable/green field
                 rank = IR_IMMUTABLE
             try:
                 mangled_name, r = self._get_field(name)
             except KeyError:
                 continue
+            if quasi and hints.get("immutable"):
+                raise TyperError(
+                    "can't have _immutable_ = True and a quasi-immutable field "
+                    "%s in class %s" % (name, self.classdef))
             ranking[mangled_name] = rank
         accessor.initialize(self.object_type, ranking)
         return ranking
diff --git a/rpython/rtyper/rvirtualizable.py b/rpython/rtyper/rvirtualizable.py
--- a/rpython/rtyper/rvirtualizable.py
+++ b/rpython/rtyper/rvirtualizable.py
@@ -38,8 +38,8 @@
             else:
                 self._super()._setup_repr(hints = hints)
             c_vfields = self.classdef.classdesc.classdict['_virtualizable_']
-            self.my_redirected_fields = self._parse_field_list(c_vfields.value,
-                                                               self.accessor)
+            self.my_redirected_fields = self._parse_field_list(
+                c_vfields.value, self.accessor, hints)
         else:
             self._super()._setup_repr()
             # ootype needs my_redirected_fields even for subclass. lltype does
diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py
--- a/rpython/rtyper/test/test_rclass.py
+++ b/rpython/rtyper/test/test_rclass.py
@@ -943,6 +943,19 @@
                 found.append(op.args[1].value)
         assert found == ['mutate_a', 'mutate_a', 'mutate_b']
 
+    def test_quasi_immutable_clashes_with_immutable(self):
+        from rpython.jit.metainterp.typesystem import deref
+        class A(object):
+            _immutable_ = True
+            _immutable_fields_ = ['a?']
+        def f():
+            a = A()
+            a.x = 42
+            a.a = 142
+            return A()
+        with py.test.raises(TyperError):
+            self.gengraph(f, [])
+
     def test_quasi_immutable_array(self):
         from rpython.jit.metainterp.typesystem import deref
         class A(object):
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -181,6 +181,7 @@
             ('all', '$(DEFAULT_TARGET)', []),
             ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Sat Aug 22 03:01:12 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 22 Aug 2015 03:01:12 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: 2to3
Message-ID: <20150822010112.6E1C21C0063@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79129:bff4896d8c5a
Date: 2015-08-22 03:01 +0200
http://bitbucket.org/pypy/pypy/changeset/bff4896d8c5a/

Log:	2to3

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -25,7 +25,7 @@
             assert s[i + 1] == 0    # 0
             assert s[i + 2] == 1    # VERSION_THREAD_ID
             assert s[i + 3] == 4    # len('pypy')
-            assert s[i + 4: i + 8] == 'pypy'
+            assert s[i + 4: i + 8] == b'pypy'
             i += 8
             while i < len(s):
                 if s[i] == 3:

From noreply at buildbot.pypy.org  Sat Aug 22 03:12:44 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Sat, 22 Aug 2015 03:12:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Avoid import in app_main.py before stdio
	is initialized.
Message-ID: <20150822011244.624721C0063@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79130:25989b3230e5
Date: 2015-08-22 03:06 +0200
http://bitbucket.org/pypy/pypy/changeset/25989b3230e5/

Log:	Avoid import in app_main.py before stdio is initialized.

	If the interpreter is run with the '-v' option specified, the import
	machinery prints messages to sys.stderr. This commit removes two
	imports before stdio initialization, which resulted in an
	AttributeError because of this.

	The imports were there to avoid issues during bootstrapping, which I
	did not yet run into after removing the imports.

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -266,16 +266,7 @@
     if hasattr(sys, 'stdin'):
         return # already initialized
 
-    # Hack to avoid recursion issues during bootstrapping: pre-import
-    # the utf-8 and latin-1 codecs
-    encerr = None
-    try:
-        import encodings.utf_8
-        import encodings.latin_1
-    except ImportError as e:
-        encerr = e
-
-    try:
+    if 1:  # keep indentation
         if encoding and ':' in encoding:
             encoding, errors = encoding.split(':', 1)
         else:
@@ -294,10 +285,6 @@
             print("Python error:  is a directory, cannot continue",
                   file=sys.stderr)
             os._exit(1)
-    finally:
-        if encerr:
-            display_exception(encerr)
-            del encerr
 
 def create_stdio(fd, writing, name, encoding, errors, unbuffered):
     import io

From noreply at buildbot.pypy.org  Sat Aug 22 09:08:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 22 Aug 2015 09:08:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: yay pass
	test_loop_invariant_mul_bridge_ovf2
Message-ID: <20150822070845.1F8A91C11EC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79131:3e5dc110b2ac
Date: 2015-08-22 09:08 +0200
http://bitbucket.org/pypy/pypy/changeset/3e5dc110b2ac/

Log:	yay pass test_loop_invariant_mul_bridge_ovf2

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -300,7 +300,7 @@
     loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
                        [mid_label] + loop_ops)
     loop.check_consistency()
-    jitcell_token.target_tokens = [mid_descr_token, start_descr]
+    jitcell_token.target_tokens = [start_descr, mid_descr_token]
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
     return start_descr
@@ -1015,13 +1015,11 @@
     call_pure_results = metainterp.call_pure_results
 
     if operations[-1].getopnum() == rop.JUMP:
-        jump_op = operations[-1]
         data = BridgeCompileData(label, operations[:],
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts,
                                  inline_short_preamble=inline_short_preamble)
     else:
-        jump_op = None
         data = SimpleCompileData(label, operations[:],
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -8,6 +8,7 @@
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp import compile
 
 class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
@@ -130,13 +131,19 @@
 
     def optimize_bridge(self, start_label, operations, call_pure_results,
                         inline_short_preamble):
-        assert inline_short_preamble
         self._check_no_forwarding([start_label.getarglist(),
                                     operations])
         info, ops = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], operations[:-1],
             call_pure_results, True)
         jump_op = operations[-1]
+        if not inline_short_preamble:
+            cell_token = jump_op.getdescr()
+            assert cell_token.target_tokens[0].virtual_state is None
+            jump_op = jump_op.copy_and_change(rop.JUMP,
+                                        descr=cell_token.target_tokens[0])
+            self.optimizer.send_extra_operation(jump_op)
+            return info, self.optimizer._newoperations[:]
         vs = self.jump_to_existing_trace(jump_op, inline_short_preamble)
         if vs is None:
             return info, self.optimizer._newoperations[:]
@@ -196,7 +203,8 @@
             self.make_equal_to(short_inputargs[i], jump_args[i])
         for op in short_ops:
             if op.is_guard():
-                op = self.replace_op_with(op, op.getopnum())
+                op = self.replace_op_with(op, op.getopnum(),
+                                          descr=compile.ResumeAtPositionDescr())
                 op.rd_snapshot = patchguardop.rd_snapshot
                 op.rd_frame_info_list = patchguardop.rd_frame_info_list
             self.optimizer.send_extra_operation(op)

From noreply at buildbot.pypy.org  Sat Aug 22 12:44:53 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 12:44:53 +0200 (CEST)
Subject: [pypy-commit] pypy default: Add a direct test for a bug. Thanks
 jerith on irc for prodding us until
Message-ID: <20150822104453.5C7031C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79132:1ed06832e512
Date: 2015-08-22 11:27 +0200
http://bitbucket.org/pypy/pypy/changeset/1ed06832e512/

Log:	Add a direct test for a bug. Thanks jerith on irc for prodding us
	until we really looked into it (and thanks undodb-gdb for being
	essential here).

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1899,7 +1899,7 @@
                 #
                 self.old_objects_pointing_to_pinned.append(parent)
                 self.updated_old_objects_pointing_to_pinned = True
-                self.header(parent).tid |= GCFLAG_PINNED
+                self.header(parent).tid |= GCFLAG_PINNED_OBJECT_PARENT_KNOWN
             #
             if hdr.tid & GCFLAG_VISITED:
                 return
@@ -2328,13 +2328,15 @@
         while self.objects_to_trace.non_empty():
             self.visit_all_objects_step(sys.maxint)
 
+    TEST_VISIT_SINGLE_STEP = False    # for tests
+
     def visit_all_objects_step(self, size_to_track):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
         while pending.non_empty():
             obj = pending.pop()
             size_to_track -= self.visit(obj)
-            if size_to_track < 0:
+            if size_to_track < 0 or self.TEST_VISIT_SINGLE_STEP:
                 return 0
         return size_to_track
 
diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -88,7 +88,7 @@
 
 class TestIncminimark(PinningGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
-    from rpython.memory.gc.incminimark import STATE_SCANNING
+    from rpython.memory.gc.incminimark import STATE_SCANNING, STATE_MARKING
 
     def test_try_pin_gcref_containing_type(self):
         # scenario: incminimark's object pinning can't pin objects that may
@@ -917,3 +917,65 @@
         py.test.raises(Exception, self.malloc, T)
     test_full_pinned_nursery_pin_fail.max_number_of_pinned_objects = 50
 
+
+    def test_pin_bug1(self):
+        #
+        # * the nursery contains a pinned object 'ptr1'
+        #
+        # * outside the nursery is another object 'ptr2' pointing to 'ptr1'
+        #
+        # * during one incremental tracing step, we see 'ptr2' but don't
+        #   trace 'ptr1' right now: it is left behind on the trace-me-later
+        #   list
+        #
+        # * then we run the program, unpin 'ptr1', and remove it from 'ptr2'
+        #
+        # * at the next minor collection, we free 'ptr1' because we don't
+        #   find anything pointing to it (it is removed from 'ptr2'),
+        #   but 'ptr1' is still in the trace-me-later list
+        #
+        # * the trace-me-later list is deep enough that 'ptr1' is not
+        #   seen right now!  it is only seen at some later minor collection
+        #
+        # * at that later point, crash, because 'ptr1' in the nursery was
+        #   overwritten
+        #
+        ptr2 = self.malloc(S)
+        ptr2.someInt = 102
+        self.stackroots.append(ptr2)
+
+        self.gc.collect()
+        ptr2 = self.stackroots[-1]    # now outside the nursery
+        adr2 = llmemory.cast_ptr_to_adr(ptr2)
+
+        ptr1 = self.malloc(T)
+        adr1 = llmemory.cast_ptr_to_adr(ptr1)
+        ptr1.someInt = 101
+        self.write(ptr2, 'data', ptr1)
+        res = self.gc.pin(adr1)
+        assert res
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_SCANNING
+        self.gc.major_collection_step()
+        assert self.gc.objects_to_trace.tolist() == [adr2]
+        assert self.gc.more_objects_to_trace.tolist() == []
+
+        self.gc.TEST_VISIT_SINGLE_STEP = True
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_MARKING
+        self.gc.major_collection_step()
+        assert self.gc.objects_to_trace.tolist() == [adr1]
+        assert self.gc.more_objects_to_trace.tolist() == [adr2]
+
+        self.write(ptr2, 'data', lltype.nullptr(T))
+        self.gc.unpin(adr1)
+
+        assert ptr1.someInt == 101
+        self.gc.minor_collection()        # should free 'ptr1'
+        py.test.raises(RuntimeError, "ptr1.someInt")
+        assert self.gc.gc_state == self.STATE_MARKING
+        self.gc.major_collection_step()   # should not crash reading 'ptr1'!
+
+        del self.gc.TEST_VISIT_SINGLE_STEP

From noreply at buildbot.pypy.org  Sat Aug 22 12:44:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 12:44:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: Try to fix the pinned objects issue
	from 1ed06832e512
Message-ID: <20150822104455.906CC1C11EC@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79133:d7187ab61ab3
Date: 2015-08-22 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/d7187ab61ab3/

Log:	Try to fix the pinned objects issue from 1ed06832e512

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -166,7 +166,7 @@
 
 # The marking phase. We walk the list 'objects_to_trace' of all gray objects
 # and mark all of the things they point to gray. This step lasts until there
-# are no more gray objects.
+# are no more gray objects.  ('objects_to_trace' never contains pinned objs.)
 STATE_MARKING = 1
 
 # here we kill all the unvisited objects
@@ -1146,6 +1146,9 @@
                       "raw_malloc_might_sweep must be empty outside SWEEPING")
 
             if self.gc_state == STATE_MARKING:
+                self.objects_to_trace.foreach(self._check_not_in_nursery, None)
+                self.more_objects_to_trace.foreach(self._check_not_in_nursery,
+                                                   None)
                 self._debug_objects_to_trace_dict1 = \
                                             self.objects_to_trace.stack2dict()
                 self._debug_objects_to_trace_dict2 = \
@@ -1156,6 +1159,10 @@
             else:
                 MovingGCBase.debug_check_consistency(self)
 
+    def _check_not_in_nursery(self, obj, ignore):
+        ll_assert(not self.is_in_nursery(obj),
+                  "'objects_to_trace' contains a nursery object")
+
     def debug_check_object(self, obj):
         # We are after a minor collection, and possibly after a major
         # collection step.  No object should be in the nursery (except
@@ -1789,6 +1796,8 @@
                 # If we're incrementally marking right now, sorry, we also
                 # need to add the object to 'more_objects_to_trace' and have
                 # it fully traced once at the end of the current marking phase.
+                ll_assert(not self.is_in_nursery(obj),
+                          "expected nursery obj in collect_cardrefs_to_nursery")
                 if self.gc_state == STATE_MARKING:
                     self.header(obj).tid &= ~GCFLAG_VISITED
                     self.more_objects_to_trace.append(obj)
@@ -1845,8 +1854,11 @@
         # need to record the not-visited-yet (white) old objects.  So
         # as a conservative approximation, we need to add the object to
         # the list if and only if it doesn't have GCFLAG_VISITED yet.
+        #
+        # Additionally, ignore pinned objects.
+        #
         obj = root.address[0]
-        if not self.header(obj).tid & GCFLAG_VISITED:
+        if (self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_PINNED)) == 0:
             self.more_objects_to_trace.append(obj)
 
     def _trace_drag_out(self, root, parent):
@@ -2033,6 +2045,7 @@
         new.delete()
 
     def _add_to_more_objects_to_trace(self, obj, ignored):
+        ll_assert(not self.is_in_nursery(obj), "unexpected nursery obj here")
         self.header(obj).tid &= ~GCFLAG_VISITED
         self.more_objects_to_trace.append(obj)
 
@@ -2287,8 +2300,7 @@
     def collect_roots(self):
         # Collect all roots.  Starts from all the objects
         # from 'prebuilt_root_objects'.
-        self.prebuilt_root_objects.foreach(self._collect_obj,
-                                           self.objects_to_trace)
+        self.prebuilt_root_objects.foreach(self._collect_obj, None)
         #
         # Add the roots from the other sources.
         self.root_walker.walk_roots(
@@ -2298,31 +2310,34 @@
         #
         # If we are in an inner collection caused by a call to a finalizer,
         # the 'run_finalizers' objects also need to be kept alive.
-        self.run_finalizers.foreach(self._collect_obj,
-                                    self.objects_to_trace)
+        self.run_finalizers.foreach(self._collect_obj, None)
 
     def enumerate_all_roots(self, callback, arg):
         self.prebuilt_root_objects.foreach(callback, arg)
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
-    @staticmethod
-    def _collect_obj(obj, objects_to_trace):
-        objects_to_trace.append(obj)
+    def _collect_obj(self, obj, ignored):
+        # Ignore pinned objects, which are the ones still in the nursery here.
+        # Cache effects: don't read any flag out of 'obj' at this point.
+        # But only checking if it is in the nursery or not is fine.
+        llop.debug_nonnull_pointer(lltype.Void, obj)
+        if not self.is_in_nursery(obj):
+            self.objects_to_trace.append(obj)
+        else:
+            # A pinned object can be found here. Such an object is handled
+            # by minor collections and shouldn't be specially handled by
+            # major collections. Therefore we only add non-pinned objects
+            # to the 'objects_to_trace' list.
+            ll_assert(self._is_pinned(obj),
+                      "non-pinned nursery obj in _collect_obj")
+    _collect_obj._always_inline_ = True
 
     def _collect_ref_stk(self, root):
-        obj = root.address[0]
-        llop.debug_nonnull_pointer(lltype.Void, obj)
-        if not self._is_pinned(obj):
-            # XXX: check if this is the right way (groggi).
-            # A pinned object can be on the stack. Such an object is handled
-            # by minor collections and shouldn't be specially handled by
-            # major collections. Therefore we only add not pinned objects to the
-            # list below.
-            self.objects_to_trace.append(obj)
+        self._collect_obj(root.address[0], None)
 
     def _collect_ref_rec(self, root, ignored):
-        self.objects_to_trace.append(root.address[0])
+        self._collect_obj(root.address[0], None)
 
     def visit_all_objects(self):
         while self.objects_to_trace.non_empty():
@@ -2351,10 +2366,17 @@
         # flag set, then the object should be in 'prebuilt_root_objects',
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
-        # Objects with GCFLAG_PINNED can't have gcptrs (see pin()), they can be
-        # ignored.
+        # We shouldn't see an object with GCFLAG_PINNED here (the pinned
+        # objects are never added to 'objects_to_trace').  The same-valued
+        # flag GCFLAG_PINNED_OBJECT_PARENT_KNOWN is used during minor
+        # collections and shouldn't be set here either.
+        #
         hdr = self.header(obj)
-        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS | GCFLAG_PINNED):
+        ll_assert((hdr.tid & GCFLAG_PINNED) == 0,
+                  "pinned object in 'objects_to_trace'")
+        ll_assert(not self.is_in_nursery(obj),
+                  "nursery object in 'objects_to_trace'")
+        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is
@@ -2584,6 +2606,7 @@
         # recursively convert objects from state 1 to state 2.
         # The call to visit_all_objects() will add the GCFLAG_VISITED
         # recursively.
+        ll_assert(not self.is_in_nursery(obj), "pinned finalizer object??")
         self.objects_to_trace.append(obj)
         self.visit_all_objects()
 
diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -966,7 +966,7 @@
         self.gc.minor_collection()
         assert self.gc.gc_state == self.STATE_MARKING
         self.gc.major_collection_step()
-        assert self.gc.objects_to_trace.tolist() == [adr1]
+        assert self.gc.objects_to_trace.tolist() == []
         assert self.gc.more_objects_to_trace.tolist() == [adr2]
 
         self.write(ptr2, 'data', lltype.nullptr(T))

From noreply at buildbot.pypy.org  Sat Aug 22 13:30:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 13:30:01 +0200 (CEST)
Subject: [pypy-commit] pypy default: Try to fix the pinned objects issue
	from 1ed06832e512
Message-ID: <20150822113001.A679C1C0359@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79134:d6887f67ca77
Date: 2015-08-22 12:44 +0200
http://bitbucket.org/pypy/pypy/changeset/d6887f67ca77/

Log:	Try to fix the pinned objects issue from 1ed06832e512

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -166,7 +166,7 @@
 
 # The marking phase. We walk the list 'objects_to_trace' of all gray objects
 # and mark all of the things they point to gray. This step lasts until there
-# are no more gray objects.
+# are no more gray objects.  ('objects_to_trace' never contains pinned objs.)
 STATE_MARKING = 1
 
 # here we kill all the unvisited objects
@@ -1146,6 +1146,9 @@
                       "raw_malloc_might_sweep must be empty outside SWEEPING")
 
             if self.gc_state == STATE_MARKING:
+                self.objects_to_trace.foreach(self._check_not_in_nursery, None)
+                self.more_objects_to_trace.foreach(self._check_not_in_nursery,
+                                                   None)
                 self._debug_objects_to_trace_dict1 = \
                                             self.objects_to_trace.stack2dict()
                 self._debug_objects_to_trace_dict2 = \
@@ -1156,6 +1159,10 @@
             else:
                 MovingGCBase.debug_check_consistency(self)
 
+    def _check_not_in_nursery(self, obj, ignore):
+        ll_assert(not self.is_in_nursery(obj),
+                  "'objects_to_trace' contains a nursery object")
+
     def debug_check_object(self, obj):
         # We are after a minor collection, and possibly after a major
         # collection step.  No object should be in the nursery (except
@@ -1789,6 +1796,8 @@
                 # If we're incrementally marking right now, sorry, we also
                 # need to add the object to 'more_objects_to_trace' and have
                 # it fully traced once at the end of the current marking phase.
+                ll_assert(not self.is_in_nursery(obj),
+                          "expected nursery obj in collect_cardrefs_to_nursery")
                 if self.gc_state == STATE_MARKING:
                     self.header(obj).tid &= ~GCFLAG_VISITED
                     self.more_objects_to_trace.append(obj)
@@ -1845,8 +1854,11 @@
         # need to record the not-visited-yet (white) old objects.  So
         # as a conservative approximation, we need to add the object to
         # the list if and only if it doesn't have GCFLAG_VISITED yet.
+        #
+        # Additionally, ignore pinned objects.
+        #
         obj = root.address[0]
-        if not self.header(obj).tid & GCFLAG_VISITED:
+        if (self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_PINNED)) == 0:
             self.more_objects_to_trace.append(obj)
 
     def _trace_drag_out(self, root, parent):
@@ -2033,6 +2045,7 @@
         new.delete()
 
     def _add_to_more_objects_to_trace(self, obj, ignored):
+        ll_assert(not self.is_in_nursery(obj), "unexpected nursery obj here")
         self.header(obj).tid &= ~GCFLAG_VISITED
         self.more_objects_to_trace.append(obj)
 
@@ -2287,8 +2300,7 @@
     def collect_roots(self):
         # Collect all roots.  Starts from all the objects
         # from 'prebuilt_root_objects'.
-        self.prebuilt_root_objects.foreach(self._collect_obj,
-                                           self.objects_to_trace)
+        self.prebuilt_root_objects.foreach(self._collect_obj, None)
         #
         # Add the roots from the other sources.
         self.root_walker.walk_roots(
@@ -2298,31 +2310,34 @@
         #
         # If we are in an inner collection caused by a call to a finalizer,
         # the 'run_finalizers' objects also need to be kept alive.
-        self.run_finalizers.foreach(self._collect_obj,
-                                    self.objects_to_trace)
+        self.run_finalizers.foreach(self._collect_obj, None)
 
     def enumerate_all_roots(self, callback, arg):
         self.prebuilt_root_objects.foreach(callback, arg)
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
-    @staticmethod
-    def _collect_obj(obj, objects_to_trace):
-        objects_to_trace.append(obj)
+    def _collect_obj(self, obj, ignored):
+        # Ignore pinned objects, which are the ones still in the nursery here.
+        # Cache effects: don't read any flag out of 'obj' at this point.
+        # But only checking if it is in the nursery or not is fine.
+        llop.debug_nonnull_pointer(lltype.Void, obj)
+        if not self.is_in_nursery(obj):
+            self.objects_to_trace.append(obj)
+        else:
+            # A pinned object can be found here. Such an object is handled
+            # by minor collections and shouldn't be specially handled by
+            # major collections. Therefore we only add non-pinned objects
+            # to the 'objects_to_trace' list.
+            ll_assert(self._is_pinned(obj),
+                      "non-pinned nursery obj in _collect_obj")
+    _collect_obj._always_inline_ = True
 
     def _collect_ref_stk(self, root):
-        obj = root.address[0]
-        llop.debug_nonnull_pointer(lltype.Void, obj)
-        if not self._is_pinned(obj):
-            # XXX: check if this is the right way (groggi).
-            # A pinned object can be on the stack. Such an object is handled
-            # by minor collections and shouldn't be specially handled by
-            # major collections. Therefore we only add not pinned objects to the
-            # list below.
-            self.objects_to_trace.append(obj)
+        self._collect_obj(root.address[0], None)
 
     def _collect_ref_rec(self, root, ignored):
-        self.objects_to_trace.append(root.address[0])
+        self._collect_obj(root.address[0], None)
 
     def visit_all_objects(self):
         while self.objects_to_trace.non_empty():
@@ -2349,10 +2364,17 @@
         # flag set, then the object should be in 'prebuilt_root_objects',
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
-        # Objects with GCFLAG_PINNED can't have gcptrs (see pin()), they can be
-        # ignored.
+        # We shouldn't see an object with GCFLAG_PINNED here (the pinned
+        # objects are never added to 'objects_to_trace').  The same-valued
+        # flag GCFLAG_PINNED_OBJECT_PARENT_KNOWN is used during minor
+        # collections and shouldn't be set here either.
+        #
         hdr = self.header(obj)
-        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS | GCFLAG_PINNED):
+        ll_assert((hdr.tid & GCFLAG_PINNED) == 0,
+                  "pinned object in 'objects_to_trace'")
+        ll_assert(not self.is_in_nursery(obj),
+                  "nursery object in 'objects_to_trace'")
+        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is
@@ -2582,6 +2604,7 @@
         # recursively convert objects from state 1 to state 2.
         # The call to visit_all_objects() will add the GCFLAG_VISITED
         # recursively.
+        ll_assert(not self.is_in_nursery(obj), "pinned finalizer object??")
         self.objects_to_trace.append(obj)
         self.visit_all_objects()
 

From noreply at buildbot.pypy.org  Sat Aug 22 13:30:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 13:30:04 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150822113004.53A3D1C0359@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79135:ef2a16781e33
Date: 2015-08-22 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/ef2a16781e33/

Log:	merge heads

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.0
+Version: 1.2.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.0"
+VERSION = "1.2.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,4 +3427,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -379,6 +379,8 @@
     PyObject *ht_name, *ht_slots;
 } PyHeapTypeObject;
 
+#define PyObject_Bytes PyObject_Str
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -160,6 +160,35 @@
     assert func.name == 'sin'
     assert func.BType == ', ), , False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -238,6 +267,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,6 +25,9 @@
     if 1:     # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
+    else:
+        kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+                                      ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -318,15 +318,32 @@
                 import cffi
                 ffi = cffi.FFI()
                 ffi.set_source("pack3.mymod", "/*code would be here*/")
+                ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
+                from distutils.command.build_ext import build_ext
+                import os
+
+                class TestBuildExt(build_ext):
+                    def pre_run(self, ext, ffi):
+                        print('_make_setuptools_api: in pre_run:', end=" ")
+                        assert ffi._hi_there == 42
+                        assert ext.name == "pack3.mymod"
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print('creating %r' % (fn,))
+                        open(fn, 'w').close()
+
                 setup(name='example1',
                       version='0.1',
                       packages=['pack3'],
                       package_dir={'': 'src1'},
-                      cffi_modules=["src1/pack3/_build.py:ffi"])
+                      cffi_modules=["src1/pack3/_build.py:ffi"],
+                      cmdclass={'build_ext': TestBuildExt},
+                      )
             """)
 
     @chdir_to_tmp
@@ -335,6 +352,7 @@
         self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None}}})
 
@@ -344,6 +362,7 @@
         self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None,
                                                       'mymod.SO': None}}})
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1911,7 +1911,7 @@
                 #
                 self.old_objects_pointing_to_pinned.append(parent)
                 self.updated_old_objects_pointing_to_pinned = True
-                self.header(parent).tid |= GCFLAG_PINNED
+                self.header(parent).tid |= GCFLAG_PINNED_OBJECT_PARENT_KNOWN
             #
             if hdr.tid & GCFLAG_VISITED:
                 return
@@ -2343,13 +2343,15 @@
         while self.objects_to_trace.non_empty():
             self.visit_all_objects_step(sys.maxint)
 
+    TEST_VISIT_SINGLE_STEP = False    # for tests
+
     def visit_all_objects_step(self, size_to_track):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
         while pending.non_empty():
             obj = pending.pop()
             size_to_track -= self.visit(obj)
-            if size_to_track < 0:
+            if size_to_track < 0 or self.TEST_VISIT_SINGLE_STEP:
                 return 0
         return size_to_track
 
diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -88,7 +88,7 @@
 
 class TestIncminimark(PinningGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
-    from rpython.memory.gc.incminimark import STATE_SCANNING
+    from rpython.memory.gc.incminimark import STATE_SCANNING, STATE_MARKING
 
     def test_try_pin_gcref_containing_type(self):
         # scenario: incminimark's object pinning can't pin objects that may
@@ -917,3 +917,65 @@
         py.test.raises(Exception, self.malloc, T)
     test_full_pinned_nursery_pin_fail.max_number_of_pinned_objects = 50
 
+
+    def test_pin_bug1(self):
+        #
+        # * the nursery contains a pinned object 'ptr1'
+        #
+        # * outside the nursery is another object 'ptr2' pointing to 'ptr1'
+        #
+        # * during one incremental tracing step, we see 'ptr2' but don't
+        #   trace 'ptr1' right now: it is left behind on the trace-me-later
+        #   list
+        #
+        # * then we run the program, unpin 'ptr1', and remove it from 'ptr2'
+        #
+        # * at the next minor collection, we free 'ptr1' because we don't
+        #   find anything pointing to it (it is removed from 'ptr2'),
+        #   but 'ptr1' is still in the trace-me-later list
+        #
+        # * the trace-me-later list is deep enough that 'ptr1' is not
+        #   seen right now!  it is only seen at some later minor collection
+        #
+        # * at that later point, crash, because 'ptr1' in the nursery was
+        #   overwritten
+        #
+        ptr2 = self.malloc(S)
+        ptr2.someInt = 102
+        self.stackroots.append(ptr2)
+
+        self.gc.collect()
+        ptr2 = self.stackroots[-1]    # now outside the nursery
+        adr2 = llmemory.cast_ptr_to_adr(ptr2)
+
+        ptr1 = self.malloc(T)
+        adr1 = llmemory.cast_ptr_to_adr(ptr1)
+        ptr1.someInt = 101
+        self.write(ptr2, 'data', ptr1)
+        res = self.gc.pin(adr1)
+        assert res
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_SCANNING
+        self.gc.major_collection_step()
+        assert self.gc.objects_to_trace.tolist() == [adr2]
+        assert self.gc.more_objects_to_trace.tolist() == []
+
+        self.gc.TEST_VISIT_SINGLE_STEP = True
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_MARKING
+        self.gc.major_collection_step()
+        assert self.gc.objects_to_trace.tolist() == []
+        assert self.gc.more_objects_to_trace.tolist() == [adr2]
+
+        self.write(ptr2, 'data', lltype.nullptr(T))
+        self.gc.unpin(adr1)
+
+        assert ptr1.someInt == 101
+        self.gc.minor_collection()        # should free 'ptr1'
+        py.test.raises(RuntimeError, "ptr1.someInt")
+        assert self.gc.gc_state == self.STATE_MARKING
+        self.gc.major_collection_step()   # should not crash reading 'ptr1'!
+
+        del self.gc.TEST_VISIT_SINGLE_STEP
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -79,10 +79,20 @@
 
     cont_name = 'rpyvmprof_f_%s_%s' % (name, token)
     tramp_name = 'rpyvmprof_t_%s_%s' % (name, token)
+    orig_tramp_name = tramp_name
 
     func.c_name = cont_name
     func._dont_inline_ = True
 
+    if sys.platform == 'darwin':
+        # according to internet "At the time UNIX was written in 1974...."
+        # "... all C functions are prefixed with _"
+        cont_name = '_' + cont_name
+        tramp_name = '_' + tramp_name
+        PLT = ""
+    else:
+        PLT = "@PLT"
+
     assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
         "rvmprof only supports x86-64 CPUs for now")
 
@@ -107,17 +117,15 @@
     target.write("""\
 \t.text
 \t.globl\t%(tramp_name)s
-\t.type\t%(tramp_name)s, @function
 %(tramp_name)s:
 \t.cfi_startproc
 \tpushq\t%(reg)s
 \t.cfi_def_cfa_offset 16
-\tcall %(cont_name)s at PLT
+\tcall %(cont_name)s%(PLT)s
 \taddq\t$8, %%rsp
 \t.cfi_def_cfa_offset 8
 \tret
 \t.cfi_endproc
-\t.size\t%(tramp_name)s, .-%(tramp_name)s
 """ % locals())
 
     def tok2cname(tok):
@@ -129,7 +137,7 @@
 
     header = 'RPY_EXTERN %s %s(%s);\n' % (
         tok2cname(restok),
-        tramp_name,
+        orig_tramp_name,
         ', '.join([tok2cname(tok) for tok in token] + ['long']))
 
     header += """\
@@ -143,7 +151,7 @@
 #endif
 #define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s
 }
-""" % (tramp_name, tramp_name, tramp_name)
+""" % (tramp_name, orig_tramp_name, tramp_name)
 
     eci = ExternalCompilationInfo(
         post_include_bits = [header],
@@ -151,7 +159,7 @@
     )
 
     return rffi.llexternal(
-        tramp_name,
+        orig_tramp_name,
         [token2lltype(tok) for tok in token] + [lltype.Signed],
         token2lltype(restok),
         compilation_info=eci,
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -52,6 +52,14 @@
 static int opened_profile(char *interp_name);
 static void flush_codes(void);
 
+#ifdef __APPLE__
+#define UNWIND_NAME "/usr/lib/system/libunwind.dylib"
+#define UNW_PREFIX "unw"
+#else
+#define UNWIND_NAME "libunwind.so"
+#define UNW_PREFIX "_ULx86_64"
+#endif
+
 RPY_EXTERN
 char *vmprof_init(int fd, double interval, char *interp_name)
 {
@@ -62,15 +70,15 @@
     if (!unw_get_reg) {
         void *libhandle;
 
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
+        if (!(libhandle = dlopen(UNWIND_NAME, RTLD_LAZY | RTLD_LOCAL)))
             goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
+        if (!(unw_get_reg = dlsym(libhandle, UNW_PREFIX "_get_reg")))
             goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
+        if (!(unw_get_proc_info = dlsym(libhandle, UNW_PREFIX "_get_proc_info")))
             goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
+        if (!(unw_init_local = dlsym(libhandle, UNW_PREFIX  "_init_local")))
             goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
+        if (!(unw_step = dlsym(libhandle, UNW_PREFIX  "_step")))
             goto error;
     }
     if (prepare_concurrent_bufs() < 0)
@@ -453,7 +461,11 @@
     close(srcfd);
 #else
     // freebsd and mac
+#if defined(__APPLE__)
+	sprintf(buf, "vmmap %d", getpid());
+#else
     sprintf(buf, "procstat -v %d", getpid());
+#endif
     FILE *srcf = popen(buf, "r");
     if (!srcf)
         return -1;
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -181,6 +181,7 @@
             ('all', '$(DEFAULT_TARGET)', []),
             ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Sat Aug 22 14:10:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 22 Aug 2015 14:10:52 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix
Message-ID: <20150822121052.F11EE1C0359@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79136:ed226eea752c
Date: 2015-08-22 09:11 +0200
http://bitbucket.org/pypy/pypy/changeset/ed226eea752c/

Log:	fix

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -343,7 +343,7 @@
 
         extra_guards = state.extra_guards
         cpu = state.cpu
-        if self.lenbound and not self.lenbound.generalization_of(other.lenbound):
+        if self.lenbound and not self.lenbound.contains_bound(other.lenbound):
             raise VirtualStatesCantMatch("length bound does not match")
 
         if self.level == LEVEL_UNKNOWN:

From noreply at buildbot.pypy.org  Sat Aug 22 14:10:55 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sat, 22 Aug 2015 14:10:55 +0200 (CEST)
Subject: [pypy-commit] pypy default: (arigo, fijal) unbreak linux asmgcc
Message-ID: <20150822121055.220291C0359@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r79137:d12dfd19fd86
Date: 2015-08-22 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/d12dfd19fd86/

Log:	(arigo, fijal) unbreak linux asmgcc

diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -90,8 +90,14 @@
         cont_name = '_' + cont_name
         tramp_name = '_' + tramp_name
         PLT = ""
+        size_decl = ""
+        type_decl = ""
     else:
         PLT = "@PLT"
+        type_decl = "\t.type\t%s, @function" % (tramp_name,)
+        size_decl = "\t.size\t%s, .-%s" % (
+            tramp_name, tramp_name)
+
 
     assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
         "rvmprof only supports x86-64 CPUs for now")
@@ -114,9 +120,12 @@
     target = udir.join('module_cache')
     target.ensure(dir=1)
     target = target.join('trampoline_%s_%s.vmprof.s' % (name, token))
+    # NOTE! the tabs in this file are absolutely essential, things
+    #       that don't start with \t are silently ignored (: WAT!?)
     target.write("""\
 \t.text
 \t.globl\t%(tramp_name)s
+%(type_decl)s
 %(tramp_name)s:
 \t.cfi_startproc
 \tpushq\t%(reg)s
@@ -126,6 +135,7 @@
 \t.cfi_def_cfa_offset 8
 \tret
 \t.cfi_endproc
+%(size_decl)s
 """ % locals())
 
     def tok2cname(tok):

From noreply at buildbot.pypy.org  Sat Aug 22 16:12:04 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 16:12:04 +0200 (CEST)
Subject: [pypy-commit] cffi default: Be more explicit
Message-ID: <20150822141204.EA7E71C13C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2255:eb32cc92fa65
Date: 2015-08-22 16:12 +0200
http://bitbucket.org/cffi/cffi/changeset/eb32cc92fa65/

Log:	Be more explicit

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -6,6 +6,12 @@
 v1.2.1
 ======
 
+Nothing changed from v1.2.0.
+
+
+v1.2.0
+======
+
 * Out-of-line mode: ``int a[][...];`` can be used to declare a structure
   field or global variable which is, simultaneously, of total length
   unknown to the C compiler (the ``a[]`` part) and each element is

From noreply at buildbot.pypy.org  Sat Aug 22 16:09:40 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 16:09:40 +0200 (CEST)
Subject: [pypy-commit] cffi default: hg merge release-1.2
Message-ID: <20150822140940.C795A1C0359@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2254:f78386adce4f
Date: 2015-08-22 16:10 +0200
http://bitbucket.org/cffi/cffi/changeset/f78386adce4f/

Log:	hg merge release-1.2

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -53,9 +53,9 @@
 
 * http://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz
 
-   - MD5: ...
+   - MD5: faca1e420e80433db409bb1bdd0a3f45
 
-   - SHA: ...
+   - SHA: f7ed014ff1602a8e81073f5356b4cafbc3f5dce1
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,7 +3,7 @@
 ======================
 
 
-v1.2.0
+v1.2.1
 ======
 
 * Out-of-line mode: ``int a[][...];`` can be used to declare a structure

From noreply at buildbot.pypy.org  Sat Aug 22 16:40:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 16:40:32 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Manually reapply commit c6e45df
Message-ID: <20150822144032.35AC51C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: py3.3
Changeset: r79138:82a89dee8adf
Date: 2015-08-22 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/82a89dee8adf/

Log:	Manually reapply commit c6e45df

diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py
--- a/lib-python/3/distutils/unixccompiler.py
+++ b/lib-python/3/distutils/unixccompiler.py
@@ -55,7 +55,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],

From noreply at buildbot.pypy.org  Sat Aug 22 16:40:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 16:40:34 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Manually reapply commit c6e45df
Message-ID: <20150822144034.503801C0557@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: py3k
Changeset: r79139:c08c77400148
Date: 2015-08-22 16:40 +0200
http://bitbucket.org/pypy/pypy/changeset/c08c77400148/

Log:	Manually reapply commit c6e45df

diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py
--- a/lib-python/3/distutils/unixccompiler.py
+++ b/lib-python/3/distutils/unixccompiler.py
@@ -55,7 +55,7 @@
     executables = {'preprocessor' : None,
                    'compiler'     : ["cc"],
                    'compiler_so'  : ["cc"],
-                   'compiler_cxx' : ["cc"],
+                   'compiler_cxx' : ["c++"],  # pypy: changed, 'cc' is bogus
                    'linker_so'    : ["cc", "-shared"],
                    'linker_exe'   : ["cc"],
                    'archiver'     : ["ar", "-cr"],

From noreply at buildbot.pypy.org  Sat Aug 22 19:31:34 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 22 Aug 2015 19:31:34 +0200 (CEST)
Subject: [pypy-commit] pypy no_boehm_libdl: close no_boehm_libdl which
 simply removed extraneous link library from Boehm gc
Message-ID: <20150822173134.A5D841C0557@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: no_boehm_libdl
Changeset: r79140:e792cd469988
Date: 2015-08-22 20:23 +0300
http://bitbucket.org/pypy/pypy/changeset/e792cd469988/

Log:	close no_boehm_libdl which simply removed extraneous link library
	from Boehm gc


From noreply at buildbot.pypy.org  Sat Aug 22 19:31:36 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 22 Aug 2015 19:31:36 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge no_boehm_libdl into default
Message-ID: <20150822173136.E34311C0557@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79141:3332c09a0ad8
Date: 2015-08-22 20:23 +0300
http://bitbucket.org/pypy/pypy/changeset/3332c09a0ad8/

Log:	merge no_boehm_libdl into default

diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -854,7 +854,7 @@
         platform = None
     else:
         library_dir = ''
-        libraries = ['gc', 'dl']
+        libraries = ['gc']
         includes=['gc/gc.h']
     eci = ExternalCompilationInfo(
         platform=platform,

From noreply at buildbot.pypy.org  Sat Aug 22 19:31:38 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 22 Aug 2015 19:31:38 +0200 (CEST)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <20150822173138.E89A21C0557@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79142:07769be4057b
Date: 2015-08-22 20:27 +0300
http://bitbucket.org/pypy/pypy/changeset/07769be4057b/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -3,7 +3,7 @@
 ========================
 
 .. this is a revision shortly after release-2.6.0
-.. startrev: 91904d5c5188
+.. startrev: 3332c09a0ad8
 
 .. branch: use_min_scalar
 Correctly resolve the output dtype of ufunc(array, scalar) calls.
@@ -70,3 +70,7 @@
 .. branch: vmprof-review
 
 Clean up of vmprof, notably to handle correctly multiple threads
+
+.. branch: no_boehm_dl
+
+Remove extra link library from Boehm GC
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,5 +3,5 @@
 =======================
 
 .. this is a revision shortly after release-2.6.1
-.. startrev: 83ebc73d4fcb
+.. startrev: 3332c09a0ad8
 

From noreply at buildbot.pypy.org  Sat Aug 22 19:31:41 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 22 Aug 2015 19:31:41 +0200 (CEST)
Subject: [pypy-commit] pypy default: bump 2.6.1 starting revision
Message-ID: <20150822173141.011C51C0557@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79143:b712ea6f504e
Date: 2015-08-22 20:29 +0300
http://bitbucket.org/pypy/pypy/changeset/b712ea6f504e/

Log:	bump 2.6.1 starting revision

diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -3,7 +3,7 @@
 ========================
 
 .. this is a revision shortly after release-2.6.0
-.. startrev: 3332c09a0ad8
+.. startrev: 91904d5c5188
 
 .. branch: use_min_scalar
 Correctly resolve the output dtype of ufunc(array, scalar) calls.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,5 +3,5 @@
 =======================
 
 .. this is a revision shortly after release-2.6.1
-.. startrev: 3332c09a0ad8
+.. startrev: 07769be4057b
 

From noreply at buildbot.pypy.org  Sat Aug 22 19:31:43 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 22 Aug 2015 19:31:43 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: merge default into release
Message-ID: <20150822173143.57B871C0557@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79144:1786177ea575
Date: 2015-08-22 20:30 +0300
http://bitbucket.org/pypy/pypy/changeset/1786177ea575/

Log:	merge default into release

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.0
+Version: 1.2.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.6.1.rst
    whatsnew-2.6.0.rst
    whatsnew-2.5.1.rst
    whatsnew-2.5.0.rst
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -32,7 +32,10 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+Part of cffi 1.2.
+
 .. branch: cffi-new-allocator
+Part of cffi 1.2.
 
 .. branch: unicode-dtype
 
@@ -67,3 +70,7 @@
 .. branch: vmprof-review
 
 Clean up of vmprof, notably to handle correctly multiple threads
+
+.. branch: no_boehm_dl
+
+Remove extra link library from Boehm GC
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,5 +3,5 @@
 =======================
 
 .. this is a revision shortly after release-2.6.1
-.. startrev: 83ebc73d4fcb
+.. startrev: 07769be4057b
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.0"
+VERSION = "1.2.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,4 +3427,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -379,6 +379,8 @@
     PyObject *ht_name, *ht_slots;
 } PyHeapTypeObject;
 
+#define PyObject_Bytes PyObject_Str
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -160,6 +160,35 @@
     assert func.name == 'sin'
     assert func.BType == ', ), , False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -238,6 +267,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,6 +25,9 @@
     if 1:     # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
+    else:
+        kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+                                      ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -318,15 +318,32 @@
                 import cffi
                 ffi = cffi.FFI()
                 ffi.set_source("pack3.mymod", "/*code would be here*/")
+                ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
+                from distutils.command.build_ext import build_ext
+                import os
+
+                class TestBuildExt(build_ext):
+                    def pre_run(self, ext, ffi):
+                        print('_make_setuptools_api: in pre_run:', end=" ")
+                        assert ffi._hi_there == 42
+                        assert ext.name == "pack3.mymod"
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print('creating %r' % (fn,))
+                        open(fn, 'w').close()
+
                 setup(name='example1',
                       version='0.1',
                       packages=['pack3'],
                       package_dir={'': 'src1'},
-                      cffi_modules=["src1/pack3/_build.py:ffi"])
+                      cffi_modules=["src1/pack3/_build.py:ffi"],
+                      cmdclass={'build_ext': TestBuildExt},
+                      )
             """)
 
     @chdir_to_tmp
@@ -335,6 +352,7 @@
         self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None}}})
 
@@ -344,6 +362,7 @@
         self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None,
                                                       'mymod.SO': None}}})
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -64,12 +64,6 @@
                                               operations,
                                               original_loop_token, log=log)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU_ARM.cast_adr_to_int(adr)
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -636,8 +636,7 @@
             assert isinstance(box, Box)
             loc = self.fm.get_new_loc(box)
             locs.append(loc.value - base_ofs)
-        if looptoken.compiled_loop_token is not None:
-            # for tests
+        if looptoken.compiled_loop_token is not None:   # <- for tests
             looptoken.compiled_loop_token._ll_initial_locs = locs
 
     def can_merge_with_next_guard(self, op, i, operations):
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -100,12 +100,6 @@
         return self.assembler.assemble_bridge(faildescr, inputargs, operations,
                                               original_loop_token, log, logger)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU386.cast_adr_to_int(adr)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -268,6 +268,7 @@
                 y -= 1
             return res
         res = self.meta_interp(f, [6, sys.maxint, 48])
+        self.check_trace_count(6)
         assert res == f(6, sys.maxint, 48)
 
     def test_loop_invariant_mul_bridge_ovf2(self):
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -166,7 +166,7 @@
 
 # The marking phase. We walk the list 'objects_to_trace' of all gray objects
 # and mark all of the things they point to gray. This step lasts until there
-# are no more gray objects.
+# are no more gray objects.  ('objects_to_trace' never contains pinned objs.)
 STATE_MARKING = 1
 
 # here we kill all the unvisited objects
@@ -1146,6 +1146,9 @@
                       "raw_malloc_might_sweep must be empty outside SWEEPING")
 
             if self.gc_state == STATE_MARKING:
+                self.objects_to_trace.foreach(self._check_not_in_nursery, None)
+                self.more_objects_to_trace.foreach(self._check_not_in_nursery,
+                                                   None)
                 self._debug_objects_to_trace_dict1 = \
                                             self.objects_to_trace.stack2dict()
                 self._debug_objects_to_trace_dict2 = \
@@ -1156,6 +1159,10 @@
             else:
                 MovingGCBase.debug_check_consistency(self)
 
+    def _check_not_in_nursery(self, obj, ignore):
+        ll_assert(not self.is_in_nursery(obj),
+                  "'objects_to_trace' contains a nursery object")
+
     def debug_check_object(self, obj):
         # We are after a minor collection, and possibly after a major
         # collection step.  No object should be in the nursery (except
@@ -1789,6 +1796,8 @@
                 # If we're incrementally marking right now, sorry, we also
                 # need to add the object to 'more_objects_to_trace' and have
                 # it fully traced once at the end of the current marking phase.
+                ll_assert(not self.is_in_nursery(obj),
+                          "expected nursery obj in collect_cardrefs_to_nursery")
                 if self.gc_state == STATE_MARKING:
                     self.header(obj).tid &= ~GCFLAG_VISITED
                     self.more_objects_to_trace.append(obj)
@@ -1845,8 +1854,11 @@
         # need to record the not-visited-yet (white) old objects.  So
         # as a conservative approximation, we need to add the object to
         # the list if and only if it doesn't have GCFLAG_VISITED yet.
+        #
+        # Additionally, ignore pinned objects.
+        #
         obj = root.address[0]
-        if not self.header(obj).tid & GCFLAG_VISITED:
+        if (self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_PINNED)) == 0:
             self.more_objects_to_trace.append(obj)
 
     def _trace_drag_out(self, root, parent):
@@ -1899,7 +1911,7 @@
                 #
                 self.old_objects_pointing_to_pinned.append(parent)
                 self.updated_old_objects_pointing_to_pinned = True
-                self.header(parent).tid |= GCFLAG_PINNED
+                self.header(parent).tid |= GCFLAG_PINNED_OBJECT_PARENT_KNOWN
             #
             if hdr.tid & GCFLAG_VISITED:
                 return
@@ -2033,6 +2045,7 @@
         new.delete()
 
     def _add_to_more_objects_to_trace(self, obj, ignored):
+        ll_assert(not self.is_in_nursery(obj), "unexpected nursery obj here")
         self.header(obj).tid &= ~GCFLAG_VISITED
         self.more_objects_to_trace.append(obj)
 
@@ -2287,8 +2300,7 @@
     def collect_roots(self):
         # Collect all roots.  Starts from all the objects
         # from 'prebuilt_root_objects'.
-        self.prebuilt_root_objects.foreach(self._collect_obj,
-                                           self.objects_to_trace)
+        self.prebuilt_root_objects.foreach(self._collect_obj, None)
         #
         # Add the roots from the other sources.
         self.root_walker.walk_roots(
@@ -2298,43 +2310,48 @@
         #
         # If we are in an inner collection caused by a call to a finalizer,
         # the 'run_finalizers' objects also need to be kept alive.
-        self.run_finalizers.foreach(self._collect_obj,
-                                    self.objects_to_trace)
+        self.run_finalizers.foreach(self._collect_obj, None)
 
     def enumerate_all_roots(self, callback, arg):
         self.prebuilt_root_objects.foreach(callback, arg)
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
-    @staticmethod
-    def _collect_obj(obj, objects_to_trace):
-        objects_to_trace.append(obj)
+    def _collect_obj(self, obj, ignored):
+        # Ignore pinned objects, which are the ones still in the nursery here.
+        # Cache effects: don't read any flag out of 'obj' at this point.
+        # But only checking if it is in the nursery or not is fine.
+        llop.debug_nonnull_pointer(lltype.Void, obj)
+        if not self.is_in_nursery(obj):
+            self.objects_to_trace.append(obj)
+        else:
+            # A pinned object can be found here. Such an object is handled
+            # by minor collections and shouldn't be specially handled by
+            # major collections. Therefore we only add non-pinned objects
+            # to the 'objects_to_trace' list.
+            ll_assert(self._is_pinned(obj),
+                      "non-pinned nursery obj in _collect_obj")
+    _collect_obj._always_inline_ = True
 
     def _collect_ref_stk(self, root):
-        obj = root.address[0]
-        llop.debug_nonnull_pointer(lltype.Void, obj)
-        if not self._is_pinned(obj):
-            # XXX: check if this is the right way (groggi).
-            # A pinned object can be on the stack. Such an object is handled
-            # by minor collections and shouldn't be specially handled by
-            # major collections. Therefore we only add not pinned objects to the
-            # list below.
-            self.objects_to_trace.append(obj)
+        self._collect_obj(root.address[0], None)
 
     def _collect_ref_rec(self, root, ignored):
-        self.objects_to_trace.append(root.address[0])
+        self._collect_obj(root.address[0], None)
 
     def visit_all_objects(self):
         while self.objects_to_trace.non_empty():
             self.visit_all_objects_step(sys.maxint)
 
+    TEST_VISIT_SINGLE_STEP = False    # for tests
+
     def visit_all_objects_step(self, size_to_track):
         # Objects can be added to pending by visit
         pending = self.objects_to_trace
         while pending.non_empty():
             obj = pending.pop()
             size_to_track -= self.visit(obj)
-            if size_to_track < 0:
+            if size_to_track < 0 or self.TEST_VISIT_SINGLE_STEP:
                 return 0
         return size_to_track
 
@@ -2349,10 +2366,17 @@
         # flag set, then the object should be in 'prebuilt_root_objects',
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
-        # Objects with GCFLAG_PINNED can't have gcptrs (see pin()), they can be
-        # ignored.
+        # We shouldn't see an object with GCFLAG_PINNED here (the pinned
+        # objects are never added to 'objects_to_trace').  The same-valued
+        # flag GCFLAG_PINNED_OBJECT_PARENT_KNOWN is used during minor
+        # collections and shouldn't be set here either.
+        #
         hdr = self.header(obj)
-        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS | GCFLAG_PINNED):
+        ll_assert((hdr.tid & GCFLAG_PINNED) == 0,
+                  "pinned object in 'objects_to_trace'")
+        ll_assert(not self.is_in_nursery(obj),
+                  "nursery object in 'objects_to_trace'")
+        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is
@@ -2582,6 +2606,7 @@
         # recursively convert objects from state 1 to state 2.
         # The call to visit_all_objects() will add the GCFLAG_VISITED
         # recursively.
+        ll_assert(not self.is_in_nursery(obj), "pinned finalizer object??")
         self.objects_to_trace.append(obj)
         self.visit_all_objects()
 
diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -88,7 +88,7 @@
 
 class TestIncminimark(PinningGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
-    from rpython.memory.gc.incminimark import STATE_SCANNING
+    from rpython.memory.gc.incminimark import STATE_SCANNING, STATE_MARKING
 
     def test_try_pin_gcref_containing_type(self):
         # scenario: incminimark's object pinning can't pin objects that may
@@ -917,3 +917,65 @@
         py.test.raises(Exception, self.malloc, T)
     test_full_pinned_nursery_pin_fail.max_number_of_pinned_objects = 50
 
+
+    def test_pin_bug1(self):
+        #
+        # * the nursery contains a pinned object 'ptr1'
+        #
+        # * outside the nursery is another object 'ptr2' pointing to 'ptr1'
+        #
+        # * during one incremental tracing step, we see 'ptr2' but don't
+        #   trace 'ptr1' right now: it is left behind on the trace-me-later
+        #   list
+        #
+        # * then we run the program, unpin 'ptr1', and remove it from 'ptr2'
+        #
+        # * at the next minor collection, we free 'ptr1' because we don't
+        #   find anything pointing to it (it is removed from 'ptr2'),
+        #   but 'ptr1' is still in the trace-me-later list
+        #
+        # * the trace-me-later list is deep enough that 'ptr1' is not
+        #   seen right now!  it is only seen at some later minor collection
+        #
+        # * at that later point, crash, because 'ptr1' in the nursery was
+        #   overwritten
+        #
+        ptr2 = self.malloc(S)
+        ptr2.someInt = 102
+        self.stackroots.append(ptr2)
+
+        self.gc.collect()
+        ptr2 = self.stackroots[-1]    # now outside the nursery
+        adr2 = llmemory.cast_ptr_to_adr(ptr2)
+
+        ptr1 = self.malloc(T)
+        adr1 = llmemory.cast_ptr_to_adr(ptr1)
+        ptr1.someInt = 101
+        self.write(ptr2, 'data', ptr1)
+        res = self.gc.pin(adr1)
+        assert res
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_SCANNING
+        self.gc.major_collection_step()
+        assert self.gc.objects_to_trace.tolist() == [adr2]
+        assert self.gc.more_objects_to_trace.tolist() == []
+
+        self.gc.TEST_VISIT_SINGLE_STEP = True
+
+        self.gc.minor_collection()
+        assert self.gc.gc_state == self.STATE_MARKING
+        self.gc.major_collection_step()
+        assert self.gc.objects_to_trace.tolist() == []
+        assert self.gc.more_objects_to_trace.tolist() == [adr2]
+
+        self.write(ptr2, 'data', lltype.nullptr(T))
+        self.gc.unpin(adr1)
+
+        assert ptr1.someInt == 101
+        self.gc.minor_collection()        # should free 'ptr1'
+        py.test.raises(RuntimeError, "ptr1.someInt")
+        assert self.gc.gc_state == self.STATE_MARKING
+        self.gc.major_collection_step()   # should not crash reading 'ptr1'!
+
+        del self.gc.TEST_VISIT_SINGLE_STEP
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -79,10 +79,26 @@
 
     cont_name = 'rpyvmprof_f_%s_%s' % (name, token)
     tramp_name = 'rpyvmprof_t_%s_%s' % (name, token)
+    orig_tramp_name = tramp_name
 
     func.c_name = cont_name
     func._dont_inline_ = True
 
+    if sys.platform == 'darwin':
+        # according to internet "At the time UNIX was written in 1974...."
+        # "... all C functions are prefixed with _"
+        cont_name = '_' + cont_name
+        tramp_name = '_' + tramp_name
+        PLT = ""
+        size_decl = ""
+        type_decl = ""
+    else:
+        PLT = "@PLT"
+        type_decl = "\t.type\t%s, @function" % (tramp_name,)
+        size_decl = "\t.size\t%s, .-%s" % (
+            tramp_name, tramp_name)
+
+
     assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
         "rvmprof only supports x86-64 CPUs for now")
 
@@ -104,20 +120,22 @@
     target = udir.join('module_cache')
     target.ensure(dir=1)
     target = target.join('trampoline_%s_%s.vmprof.s' % (name, token))
+    # NOTE! the tabs in this file are absolutely essential, things
+    #       that don't start with \t are silently ignored (: WAT!?)
     target.write("""\
 \t.text
 \t.globl\t%(tramp_name)s
-\t.type\t%(tramp_name)s, @function
+%(type_decl)s
 %(tramp_name)s:
 \t.cfi_startproc
 \tpushq\t%(reg)s
 \t.cfi_def_cfa_offset 16
-\tcall %(cont_name)s at PLT
+\tcall %(cont_name)s%(PLT)s
 \taddq\t$8, %%rsp
 \t.cfi_def_cfa_offset 8
 \tret
 \t.cfi_endproc
-\t.size\t%(tramp_name)s, .-%(tramp_name)s
+%(size_decl)s
 """ % locals())
 
     def tok2cname(tok):
@@ -129,7 +147,7 @@
 
     header = 'RPY_EXTERN %s %s(%s);\n' % (
         tok2cname(restok),
-        tramp_name,
+        orig_tramp_name,
         ', '.join([tok2cname(tok) for tok in token] + ['long']))
 
     header += """\
@@ -143,7 +161,7 @@
 #endif
 #define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s
 }
-""" % (tramp_name, tramp_name, tramp_name)
+""" % (tramp_name, orig_tramp_name, tramp_name)
 
     eci = ExternalCompilationInfo(
         post_include_bits = [header],
@@ -151,7 +169,7 @@
     )
 
     return rffi.llexternal(
-        tramp_name,
+        orig_tramp_name,
         [token2lltype(tok) for tok in token] + [lltype.Signed],
         token2lltype(restok),
         compilation_info=eci,
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
--- a/rpython/rlib/rvmprof/src/vmprof_getpc.h
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -53,7 +53,9 @@
 // If #define _GNU_SOURCE causes problems, this might work instead.
 // It will cause problems for FreeBSD though!, because it turns off
 // the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
+#ifdef __APPLE__
+#define _XOPEN_SOURCE 500
+#endif
 
 #include          // for memcmp
 #if defined(HAVE_SYS_UCONTEXT_H)
@@ -179,7 +181,11 @@
 // configure.ac (or set it manually in your config.h).
 #else
 void* GetPC(ucontext_t *signal_ucontext) {
+#ifdef __APPLE__
+  return (void*)(signal_ucontext->uc_mcontext->__ss.__rip);
+#else
   return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
+#endif
 }
 
 #endif
diff --git a/rpython/rlib/rvmprof/src/vmprof_main.h b/rpython/rlib/rvmprof/src/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/vmprof_main.h
@@ -27,6 +27,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 #include 
 #include "vmprof_getpc.h"
@@ -51,6 +52,14 @@
 static int opened_profile(char *interp_name);
 static void flush_codes(void);
 
+#ifdef __APPLE__
+#define UNWIND_NAME "/usr/lib/system/libunwind.dylib"
+#define UNW_PREFIX "unw"
+#else
+#define UNWIND_NAME "libunwind.so"
+#define UNW_PREFIX "_ULx86_64"
+#endif
+
 RPY_EXTERN
 char *vmprof_init(int fd, double interval, char *interp_name)
 {
@@ -61,15 +70,15 @@
     if (!unw_get_reg) {
         void *libhandle;
 
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
+        if (!(libhandle = dlopen(UNWIND_NAME, RTLD_LAZY | RTLD_LOCAL)))
             goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
+        if (!(unw_get_reg = dlsym(libhandle, UNW_PREFIX "_get_reg")))
             goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
+        if (!(unw_get_proc_info = dlsym(libhandle, UNW_PREFIX "_get_proc_info")))
             goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
+        if (!(unw_init_local = dlsym(libhandle, UNW_PREFIX  "_init_local")))
             goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
+        if (!(unw_step = dlsym(libhandle, UNW_PREFIX  "_step")))
             goto error;
     }
     if (prepare_concurrent_bufs() < 0)
@@ -452,12 +461,16 @@
     close(srcfd);
 #else
     // freebsd and mac
+#if defined(__APPLE__)
+	sprintf(buf, "vmmap %d", getpid());
+#else
     sprintf(buf, "procstat -v %d", getpid());
+#endif
     FILE *srcf = popen(buf, "r");
     if (!srcf)
         return -1;
 
-    while ((size = fread(buf, 1, sizeof buf, src))) {
+    while ((size = fread(buf, 1, sizeof buf, srcf))) {
         if (_write_all(buf, size) < 0) {
             pclose(srcf);
             return -1;
diff --git a/rpython/rlib/rvmprof/src/vmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h
--- a/rpython/rlib/rvmprof/src/vmprof_mt.h
+++ b/rpython/rlib/rvmprof/src/vmprof_mt.h
@@ -66,7 +66,7 @@
     unprepare_concurrent_bufs();
     profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
                                PROT_READ | PROT_WRITE,
-                               MAP_PRIVATE | MAP_ANONYMOUS,
+                               MAP_PRIVATE | MAP_ANON,
                                -1, 0);
     if (profbuf_all_buffers == MAP_FAILED) {
         profbuf_all_buffers = NULL;
diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py
--- a/rpython/rlib/rvmprof/test/test_ztranslation.py
+++ b/rpython/rlib/rvmprof/test/test_ztranslation.py
@@ -1,5 +1,6 @@
 import time, os, sys
-sys.path += ['.'] # for subprocess in test_interpreted
+if __name__ == '__main__':
+    sys.path += ['../../../..']    # for subprocess in test_interpreted
 import py
 from rpython.tool.udir import udir
 from rpython.rlib import rvmprof
@@ -59,7 +60,8 @@
 def test_interpreted():
     # takes forever if the Python process is already big...
     import subprocess
-    subprocess.check_call([sys.executable, __file__])
+    subprocess.check_call([sys.executable, os.path.basename(__file__)],
+                          cwd=(os.path.dirname(__file__) or '.'))
 
 def test_compiled():
     fn = compile(main, [], gcpolicy="minimark")
diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py
--- a/rpython/rtyper/tool/rffi_platform.py
+++ b/rpython/rtyper/tool/rffi_platform.py
@@ -854,7 +854,7 @@
         platform = None
     else:
         library_dir = ''
-        libraries = ['gc', 'dl']
+        libraries = ['gc']
         includes=['gc/gc.h']
     eci = ExternalCompilationInfo(
         platform=platform,
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -181,6 +181,7 @@
             ('all', '$(DEFAULT_TARGET)', []),
             ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'),
             ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
+            ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'),
             ]
 

From noreply at buildbot.pypy.org  Sat Aug 22 20:10:23 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 22 Aug 2015 20:10:23 +0200 (CEST)
Subject: [pypy-commit] buildbot default: Add another hack on top of the
	Mercurial class
Message-ID: <20150822181023.804281C0359@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r961:a94291b1ae85
Date: 2015-08-22 20:11 +0200
http://bitbucket.org/pypy/buildbot/changeset/a94291b1ae85/

Log:	Add another hack on top of the Mercurial class

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -273,6 +273,21 @@
         self.finished(SUCCESS)
 
 
+# hack the Mercurial class in-place: it should do "hg pull" without
+# passing a "--rev" argument.  The problem is that while it sounds like
+# a good idea, passing a "--rev" argument here changes the order of
+# the checkouts.  Then our revisions "12345:432bcbb1ba" are bogus.
+def _my_pullUpdate(self, res):
+    command = ['pull' , self.repourl]
+    #if self.revision:                   
+    #    command.extend(['--rev', self.revision])
+    d = self._dovccmd(command)
+    d.addCallback(self._checkBranchChange)
+    return d
+assert hasattr(Mercurial, '_pullUpdate')
+Mercurial._pullUpdate = _my_pullUpdate
+
+
 def update_hg_old_method(platform, factory, repourl, workdir):
     # baaaaaah.  Seems that the Mercurial class doesn't support
     # updating to a different branch than the one specified by

From noreply at buildbot.pypy.org  Sat Aug 22 20:52:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 22 Aug 2015 20:52:11 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: try to document 1ed06832e512
Message-ID: <20150822185211.95A6E1C205E@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79145:363869282f0e
Date: 2015-08-22 21:52 +0300
http://bitbucket.org/pypy/pypy/changeset/363869282f0e/

Log:	try to document 1ed06832e512

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.6.1.rst
    release-2.6.0.rst
    release-2.5.1.rst
    release-2.5.0.rst
diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -79,6 +79,8 @@
 
   * Drop support for ancient macOS v10.4 and before
 
+  * Clear up contention in the garbage collector between trace-me-later and pinning
+
   * Issues reported with our previous release were resolved_ after reports from users on
     our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
     #pypy.

From noreply at buildbot.pypy.org  Sun Aug 23 10:04:43 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 10:04:43 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: clenaup input arguments and
	fix some tests
Message-ID: <20150823080443.4D9FA1C0FF7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79146:7d5e249ccc68
Date: 2015-08-23 10:04 +0200
http://bitbucket.org/pypy/pypy/changeset/7d5e249ccc68/

Log:	clenaup input arguments and fix some tests

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -573,7 +573,7 @@
         cf = None
         if indexb.is_constant():
             index = indexb.getint()
-            arrayinfo.getlenbound().make_gt_const(index)
+            arrayinfo.getlenbound(None).make_gt_const(index)
             # use the cache on (arraydescr, index), which is a constant
             cf = self.arrayitem_cache(op.getdescr(), index)
             field = cf.getfield_from_cache(self, arrayinfo, op.getdescr())
@@ -602,7 +602,7 @@
         cf = None
         if indexb.is_constant():
             index = indexb.getint()
-            arrayinfo.getlenbound().make_gt_const(index)
+            arrayinfo.getlenbound(None).make_gt_const(index)
             # use the cache on (arraydescr, index), which is a constant
             cf = self.arrayitem_cache(op.getdescr(), index)
             fieldvalue = cf.getfield_from_cache(self, arrayinfo, op.getdescr())
@@ -631,7 +631,7 @@
         if indexb.is_constant():
             arrayinfo = self.ensure_ptr_info_arg0(op)
             # arraybound
-            arrayinfo.getlenbound().make_gt_const(indexb.getint())
+            arrayinfo.getlenbound(None).make_gt_const(indexb.getint())
             cf = self.arrayitem_cache(op.getdescr(), indexb.getint())
             cf.do_setfield(self, op)
         else:
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -38,7 +38,7 @@
     def get_known_class(self, cpu):
         return None
 
-    def getlenbound(self):
+    def getlenbound(self, mode):
         return None
 
     def getnullness(self):
@@ -331,9 +331,10 @@
             self.lenbound = intutils.ConstIntBound(size)
         self._clear = clear
 
-    def getlenbound(self):
+    def getlenbound(self, mode):
         from rpython.jit.metainterp.optimizeopt import intutils
-        
+
+        assert mode is None
         if self.lenbound is None:
             assert self.length == -1
             self.lenbound = intutils.IntLowerBound(0)
@@ -518,9 +519,9 @@
         info = self._get_array_info(optheap)
         return info.getitem(index)
 
-    def setitem(self, index, op, cf=None, optheap=None):
+    def setitem(self, index, struct, op, cf=None, optheap=None):
         info = self._get_array_info(optheap)
-        info.setitem(index, op, cf)
+        info.setitem(index, struct, op, cf)
 
     def setfield(self, descr, struct, op, optheap=None, cf=None):
         info = self._get_info(optheap)
@@ -563,9 +564,13 @@
         return self._unpack_str(mode)
 
     def getlenbound(self, mode):
-        from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound
-        
-        return ConstIntBound(self.getstrlen(None, None, mode).getint())
+        from rpython.jit.metainterp.optimizeopt.intutils import ConstIntBound,\
+                IntLowerBound
+
+        if mode is None:
+            return IntLowerBound(0)
+        else:
+            return ConstIntBound(self.getstrlen(None, None, mode).getint())
     
     def getstrlen(self, op, string_optimizer, mode, create_ops=True):
         from rpython.jit.metainterp.optimizeopt import vstring
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -411,7 +411,7 @@
     def optimize_ARRAYLEN_GC(self, op):
         array = self.ensure_ptr_info_arg0(op)
         self.emit_operation(op)
-        self.optimizer.setintbound(op, array.getlenbound())
+        self.optimizer.setintbound(op, array.getlenbound(None))
 
     def optimize_STRLEN(self, op):
         self.emit_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -219,9 +219,6 @@
     def export_state(self, start_label, original_label_args, renamed_inputargs):
         end_args = [self.get_box_replacement(a) for a in original_label_args]
         virtual_state = self.get_virtual_state(end_args)
-        inparg_mapping = [(start_label.getarg(i), end_args[i])
-                          for i in range(len(end_args)) if
-                          start_label.getarg(i) is not end_args[i]]
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
@@ -240,15 +237,16 @@
                 infos[op] = self.optimizer.getinfo(op)
         self.optimizer._clean_optimization_info(end_args)
         self.optimizer._clean_optimization_info(start_label.getarglist())
-        return ExportedState(label_args, inparg_mapping, virtual_state, infos,
+        return ExportedState(label_args, end_args, virtual_state, infos,
                              short_boxes, renamed_inputargs,
                              short_inputargs)
 
     def import_state(self, targetop, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
-        assert len(exported_state.inputarg_mapping) == len(targetop.getarglist())
-        for i, (s, target) in enumerate(exported_state.inputarg_mapping):
+        assert (len(exported_state.next_iteration_args) ==
+                len(targetop.getarglist()))
+        for i, target in enumerate(exported_state.next_iteration_args):
             source = targetop.getarg(i)
             assert source is not target
             source.set_forwarded(target)
@@ -296,9 +294,7 @@
 class ExportedState(LoopInfo):
     """ Exported state consists of a few pieces of information:
 
-    * inputarg_mapping - a list of tuples with original inputarg box
-                         as the first element and the second element being
-                         what it maps to (potentially const)
+    * next_iteration_args - starting arguments for next iteration
     * exported_infos - a mapping from ops to infos, including inputargs
     * end_args - arguments that end up in the label leading to the next
                  iteration
@@ -309,11 +305,11 @@
     * short_inputargs - the renamed inputargs for short preamble
     """
     
-    def __init__(self, end_args, inputarg_mapping, virtual_state,
+    def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
                  short_inputargs):
         self.end_args = end_args
-        self.inputarg_mapping = inputarg_mapping
+        self.next_iteration_args = next_iteration_args
         self.virtual_state = virtual_state
         self.exported_infos = exported_infos
         self.short_boxes = short_boxes
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -319,7 +319,8 @@
                     self.level = LEVEL_KNOWNCLASS
                 elif info.is_nonnull():
                     self.level = LEVEL_NONNULL
-                self.lenbound = info.getlenbound()
+                # XXX strings?
+                self.lenbound = info.getlenbound(None)
         elif type == 'i':
             self.intbound = info
 
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -914,7 +914,7 @@
             return n
         res = self.meta_interp(f, [20, 1, 2])
         assert res == 0
-        self.check_resops(call=0)
+        self.check_resops(call_i=0, call_r=0)
 
     def test_abs(self):
         myjitdriver = JitDriver(greens = [], reds = ['i', 't'])

From noreply at buildbot.pypy.org  Sun Aug 23 11:11:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 11:11:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: whack at problems some more
Message-ID: <20150823091137.B6C631C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79147:e977efbc9001
Date: 2015-08-23 10:59 +0200
http://bitbucket.org/pypy/pypy/changeset/e977efbc9001/

Log:	whack at problems some more

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -483,19 +483,17 @@
                                                 call_pure_results)
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
         preamble_data.forget_optimization_info()
-        end_label = ResOperation(rop.LABEL, start_state.end_args)
-        loop_data = compile.UnrolledLoopData(end_label, jump_op,
+        loop_data = compile.UnrolledLoopData(start_label, jump_op,
                                              ops, start_state,
                                              call_pure_results)
         loop_info, ops = self._do_optimize_loop(loop_data)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
         start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
-        inputargs = loop_info.label_args[:]
-        emit_end_label = ResOperation(rop.LABEL, inputargs)
+        emit_end_label = ResOperation(rop.LABEL, loop_info.label_args)
         preamble.operations = ([start_label] + preamble_ops +
                                loop_info.extra_same_as + [emit_end_label])
-        loop.inputargs = inputargs
+        loop.inputargs = loop_info.label_args[:]
         loop.operations = [emit_end_label] + ops
         return Info(preamble, loop_info.short_preamble,
                     start_state.virtual_state)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -49,7 +49,7 @@
                 self.make_constant_class(op, known_class, False)
             if isinstance(preamble_info, info.ArrayPtrInfo):
                 arr_info = info.ArrayPtrInfo(None)
-                arr_info.lenbound = preamble_info.getlenbound()
+                arr_info.lenbound = preamble_info.getlenbound(None)
                 op.set_forwarded(arr_info)
             if preamble_info.is_nonnull():
                 self.make_nonnull(op)
@@ -103,6 +103,8 @@
         self._check_no_forwarding([[start_label, end_jump], ops])
         self.import_state(start_label, state)
         self.potential_extra_ops = {}
+        label_args = state.virtual_state.make_inputargs(
+            start_label.getarglist(), self.optimizer)
         self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops,
                                              call_pure_results, False)
         orig_jump_args = [self.get_box_replacement(op)
@@ -120,7 +122,7 @@
                                 self.optimizer.patchguardop)
         # remove duplicates, removes stuff from used boxes too
         label_args, jump_args = self.filter_extra_jump_args(
-            start_label.getarglist() + self.short_preamble_producer.used_boxes,
+            label_args + self.short_preamble_producer.used_boxes,
             jump_args + extra_jump_args)
         jump_op = ResOperation(rop.JUMP, jump_args)
         self.optimizer.send_extra_operation(jump_op)
@@ -250,7 +252,7 @@
             source = targetop.getarg(i)
             assert source is not target
             source.set_forwarded(target)
-            info = exported_state.exported_infos.get(source, None)
+            info = exported_state.exported_infos.get(target, None)
             if info is not None:
                 self.optimizer.setinfo_from_preamble(source, info,
                                             exported_state.exported_infos)

From noreply at buildbot.pypy.org  Sun Aug 23 11:11:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 11:11:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: reenable inputargs checking
	(maybe?)
Message-ID: <20150823091139.B97251C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79148:d8b3b12ad45c
Date: 2015-08-23 11:03 +0200
http://bitbucket.org/pypy/pypy/changeset/d8b3b12ad45c/

Log:	reenable inputargs checking (maybe?)

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -340,7 +340,6 @@
         return op
 
     def is_inputarg(self, op):
-        return True
         return op in self.inparg_dict
 
     def get_constant_box(self, box):
@@ -483,9 +482,9 @@
                 new_arg = OpHelpers.inputarg_from_tp(inparg.type)
                 inparg.set_forwarded(new_arg)
                 newargs.append(new_arg)
+            self.init_inparg_dict_from(newargs)
         else:
             newargs = inputargs
-        self.init_inparg_dict_from(newargs)
         self.call_pure_results = call_pure_results
         if ops[-1].getopnum() in (rop.FINISH, rop.JUMP):
             last = len(ops) - 1
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -105,6 +105,7 @@
         self.potential_extra_ops = {}
         label_args = state.virtual_state.make_inputargs(
             start_label.getarglist(), self.optimizer)
+        self.optimizer.init_inparg_dict_from(label_args)
         self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops,
                                              call_pure_results, False)
         orig_jump_args = [self.get_box_replacement(op)

From noreply at buildbot.pypy.org  Sun Aug 23 11:11:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 11:11:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150823091141.BE31A1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79149:1643091d8b44
Date: 2015-08-23 11:11 +0200
http://bitbucket.org/pypy/pypy/changeset/1643091d8b44/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -140,8 +140,8 @@
             return fw
         return None
 
-    def get_box_replacement(self, op):
-        return self.optimizer.get_box_replacement(op)
+    def get_box_replacement(self, op, not_const=False):
+        return self.optimizer.get_box_replacement(op, not_const=not_const)
 
     def getlastop(self):
         return self.optimizer._last_emitted_op
@@ -318,10 +318,10 @@
             if self.get_box_replacement(op).is_constant():
                 return info.FloatConstInfo(self.get_box_replacement(op))
 
-    def get_box_replacement(self, op):
+    def get_box_replacement(self, op, not_const=False):
         if op is None:
             return op
-        return op.get_box_replacement()
+        return op.get_box_replacement(not_const)
 
     def force_box(self, op):
         op = self.get_box_replacement(op)
@@ -551,6 +551,9 @@
             else:
                 guard_op = self.replace_op_with(op, op.getopnum())
                 op = self.store_final_boxes_in_guard(guard_op, pendingfields)
+                # for unrolling
+                for farg in op.getfailargs():
+                    self.force_box(farg)
         elif op.can_raise():
             self.exception_might_have_happened = True
         self._really_emitted_operation = op
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -182,7 +182,7 @@
         return virtual_state
 
     def filter_extra_jump_args(self, label_args, jump_args):
-        label_args = [self.get_box_replacement(x) for x in label_args]
+        label_args = [self.get_box_replacement(x, True) for x in label_args]
         jump_args = [self.get_box_replacement(x) for x in jump_args]
         new_label_args = []
         new_jump_args = []
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -33,11 +33,12 @@
         llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
-    def get_box_replacement(op):
+    def get_box_replacement(op, not_const=False):
         orig_op = op
         c = 0
         while (op.get_forwarded() is not None and
-               not op.get_forwarded().is_info_class):
+               not op.get_forwarded().is_info_class and
+               (not not_const or not op.get_forwarded().is_constant())):
             c += 1
             op = op.get_forwarded()
         if op is not orig_op and c > 1:

From noreply at buildbot.pypy.org  Sun Aug 23 12:25:47 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 12:25:47 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: updated failure for accum to be
	printed at runtime
Message-ID: <20150823102547.0F45C1C118C@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79150:b06be6fc7944
Date: 2015-08-23 08:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b06be6fc7944/

Log:	updated failure for accum to be printed at runtime

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -346,8 +346,10 @@
             count += 1
             accum_info = accum_info.prev
         else:
-            llop.debug_print(lltype.Void, "[accumulator] %d accumulators, none matched box %s\n" % (count, accum_info.box))
-            assert 0
+            msg = "[accumulator] %d accumulators, none matched box %s\n" % (count, accum_info.box)
+            if we_are_translated():
+                llop.debug_print(lltype.Void, msg)
+            raise NotImplementedError(msg)
 
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):
         faillocs = self.locs_for_fail(guard_op)

From noreply at buildbot.pypy.org  Sun Aug 23 12:29:25 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 12:29:25 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: used not_implemented function
 already present in regalloc.py
Message-ID: <20150823102925.C5F2A1C20A7@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79151:12bbf27bd64c
Date: 2015-08-23 12:29 +0200
http://bitbucket.org/pypy/pypy/changeset/12bbf27bd64c/

Log:	used not_implemented function already present in regalloc.py

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -345,11 +345,9 @@
                 break
             count += 1
             accum_info = accum_info.prev
-        else:
-            msg = "[accumulator] %d accumulators, none matched box %s\n" % (count, accum_info.box)
-            if we_are_translated():
-                llop.debug_print(lltype.Void, msg)
-            raise NotImplementedError(msg)
+        if count == 0:
+            msg = "[accumulator] %d accumulators, none matched box %s" % (count, accum_info.box)
+            not_implemented(msg)
 
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):
         faillocs = self.locs_for_fail(guard_op)

From noreply at buildbot.pypy.org  Sun Aug 23 12:46:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 12:46:16 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: write a test for bridge
	(failing so far)
Message-ID: <20150823104616.257E51C11FD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79152:b04b6f538add
Date: 2015-08-23 12:46 +0200
http://bitbucket.org/pypy/pypy/changeset/b04b6f538add/

Log:	write a test for bridge (failing so far)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -104,6 +104,10 @@
     def as_vtable_size_descr(self):
         return self
 
+    def get_vtable(self):
+        return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
+            self._corresponding_vtable))
+
     def count_fields_if_immutable(self):
         return heaptracker.count_fields_if_immutable(self.S)
 
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -223,7 +223,7 @@
     jump_op = ops[-1]
     target_token = TargetToken(jitcell_token)
     target_token.original_jitcell_token = jitcell_token
-    label.setdescr(target_token)
+    label = ResOperation(rop.LABEL, loop_info.inputargs[:], descr=target_token)
     jump_op.setdescr(target_token)
     loop.operations = [label] + ops
     loop.check_consistency()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -94,3 +94,25 @@
         jump(i0, i1, i2, i3)
         """
         self.optimize(loop, bridge, expected)
+
+    def test_virtual_state_in_bridge(self):
+        loop = """
+        [i0, p1]
+        p0 = new_with_vtable(descr=simpledescr)
+        setfield_gc(p0, i0, descr=simplevalue)
+        i3 = int_is_true(i0)
+        guard_true(i3) [p0]
+        i1 = int_add(i0, 1)
+        jump(i1, p0)
+        """
+        bridge = """
+        [p0]
+        p1 = new_with_vtable(descr=simpledescr)
+        setfield_gc(p1, 3, descr=simplevalue)
+        jump(1, p1)
+        """
+        expected = """
+        []
+        jump(1, 3)
+        """
+        self.optimize(loop, bridge, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8253,7 +8253,7 @@
         [p22, p18]
         i1 = getfield_gc_i(p22, descr=valuedescr)
         guard_value(i1, 2) []
-        jump(p22, p18)
+        jump()
         """
         expected = """
         [p22, p18]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -212,6 +212,12 @@
                         ('one', lltype.Ptr(lltype.GcArray(lltype.Ptr(NODE)))))
     u_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     u_vtable_adr = llmemory.cast_ptr_to_adr(u_vtable)
+    SIMPLE = lltype.GcStruct('simple',
+        ('parent', OBJECT),
+        ('value', lltype.Signed))
+    simpledescr = cpu.sizeof(SIMPLE, True)
+    simplevalue = cpu.fielddescrof(SIMPLE, 'value')
+    simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     usize = cpu.sizeof(U, True)
     onedescr = cpu.fielddescrof(U, 'one')
 
@@ -350,6 +356,7 @@
     register_known_gctype(cpu, node_vtable2, NODE2)
     register_known_gctype(cpu, node_vtable3, NODE3)
     register_known_gctype(cpu, u_vtable,     U)
+    register_known_gctype(cpu, simple_vtable,     SIMPLE)
     register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
     register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
     register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -140,8 +140,8 @@
             start_label.getarglist()[:], operations[:-1],
             call_pure_results, True)
         jump_op = operations[-1]
-        if not inline_short_preamble:
-            cell_token = jump_op.getdescr()
+        cell_token = jump_op.getdescr()
+        if not inline_short_preamble or len(cell_token.target_tokens) == 1:
             assert cell_token.target_tokens[0].virtual_state is None
             jump_op = jump_op.copy_and_change(rop.JUMP,
                                         descr=cell_token.target_tokens[0])
@@ -172,7 +172,11 @@
             except VirtualStatesCantMatch:
                 continue
             short_preamble = target_token.short_preamble
-            extra = self.inline_short_preamble(args,
+            pass_to_short = target_virtual_state.make_inputargs(args,
+                self.optimizer, force_boxes=True, append_virtuals=True)
+            args = target_virtual_state.make_inputargs(args,
+                self.optimizer)
+            extra = self.inline_short_preamble(pass_to_short,
                 short_preamble[0].getarglist(), short_preamble[1:-1],
                 short_preamble[-1].getarglist(), self.optimizer.patchguardop)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -2,7 +2,8 @@
 from rpython.jit.metainterp.history import (ConstInt, Const,
         ConstPtr, ConstFloat)
 from rpython.jit.metainterp.optimizeopt import info
-from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded
+from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded,\
+     ConstIntBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
@@ -24,6 +25,8 @@
 def get_forwarded(box):
     if not isinstance(box, Const):
         return box.get_forwarded()
+    if box.type == 'i':
+        return ConstIntBound(box.getint())
     xxx
 
 class GenerateGuardState(object):
@@ -627,7 +630,8 @@
         return NotVirtualStateInfo(self.optimizer.cpu, box.type,
                                    self.optimizer.getinfo(box))
 
-    def visit_virtual(self, known_class, fielddescrs):
+    def visit_virtual(self, descr, fielddescrs):
+        known_class = ConstInt(descr.get_vtable())
         return VirtualStateInfo(known_class, fielddescrs)
 
     def visit_vstruct(self, typedescr, fielddescrs):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -1118,7 +1118,7 @@
             return x
         res = self.meta_interp(f, [20], enable_opts='')
         assert res == f(20)
-        self.check_resops(call=0)
+        self.check_resops(call_i=0, call_r=0)
 
     def test_zerodivisionerror(self):
         # test the case of exception-raising operation that is not delegated
diff --git a/rpython/jit/metainterp/walkvirtual.py b/rpython/jit/metainterp/walkvirtual.py
--- a/rpython/jit/metainterp/walkvirtual.py
+++ b/rpython/jit/metainterp/walkvirtual.py
@@ -5,7 +5,7 @@
     def visit_not_virtual(self, value):
         raise NotImplementedError("abstract base class")
 
-    def visit_virtual(self, known_class, fielddescrs):
+    def visit_virtual(self, descr, fielddescrs):
         raise NotImplementedError("abstract base class")
 
     def visit_vstruct(self, typedescr, fielddescrs):

From noreply at buildbot.pypy.org  Sun Aug 23 12:46:56 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 12:46:56 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: pass the test,
	it's silly but would do for now
Message-ID: <20150823104656.75A041C11FD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79153:aaf39aff04e1
Date: 2015-08-23 12:47 +0200
http://bitbucket.org/pypy/pypy/changeset/aaf39aff04e1/

Log:	pass the test, it's silly but would do for now

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -112,7 +112,7 @@
         jump(1, p1)
         """
         expected = """
-        []
+        [p0]
         jump(1, 3)
         """
         self.optimize(loop, bridge, expected)

From noreply at buildbot.pypy.org  Sun Aug 23 12:48:09 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 12:48:09 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150823104809.DD0DA1C11FD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79154:c52e9297ac8c
Date: 2015-08-23 12:48 +0200
http://bitbucket.org/pypy/pypy/changeset/c52e9297ac8c/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -553,7 +553,8 @@
                 op = self.store_final_boxes_in_guard(guard_op, pendingfields)
                 # for unrolling
                 for farg in op.getfailargs():
-                    self.force_box(farg)
+                    if farg:
+                        self.force_box(farg)
         elif op.can_raise():
             self.exception_might_have_happened = True
         self._really_emitted_operation = op

From noreply at buildbot.pypy.org  Sun Aug 23 13:04:47 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 13:04:47 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: logical error,
 break did not increment and leads into failure straight away
Message-ID: <20150823110447.803DB1C217F@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79155:d7bbffc00e28
Date: 2015-08-23 13:05 +0200
http://bitbucket.org/pypy/pypy/changeset/d7bbffc00e28/

Log:	logical error, break did not increment and leads into failure
	straight away

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -345,8 +345,10 @@
                 break
             count += 1
             accum_info = accum_info.prev
-        if count == 0:
+        else:
             msg = "[accumulator] %d accumulators, none matched box %s" % (count, accum_info.box)
+            print msg
+            import pdb; pdb.set_trace()
             not_implemented(msg)
 
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):

From noreply at buildbot.pypy.org  Sun Aug 23 15:14:45 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 15:14:45 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: saving of accum state more explicit
Message-ID: <20150823131445.0707A1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79156:caedfb3ee99b
Date: 2015-08-23 15:14 +0200
http://bitbucket.org/pypy/pypy/changeset/caedfb3ee99b/

Log:	saving of accum state more explicit

diff --git a/rpython/jit/backend/llsupport/assembler.py b/rpython/jit/backend/llsupport/assembler.py
--- a/rpython/jit/backend/llsupport/assembler.py
+++ b/rpython/jit/backend/llsupport/assembler.py
@@ -3,7 +3,7 @@
 from rpython.jit.backend.llsupport.symbolic import WORD
 from rpython.jit.backend.llsupport.codemap import CodemapBuilder
 from rpython.jit.metainterp.history import (INT, REF, FLOAT, VECTOR,
-    JitCellToken, ConstInt, BoxInt, AbstractFailDescr)
+    JitCellToken, ConstInt, BoxInt, AbstractFailDescr, BoxVector)
 from rpython.jit.metainterp.resoperation import ResOperation, rop
 from rpython.rlib import rgc
 from rpython.rlib.debug import (debug_start, debug_stop, have_debug_prints_for,
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -10,7 +10,6 @@
 from rpython.jit.metainterp.history import (Const, Box, VOID,
     BoxVector, ConstInt)
 from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
-from rpython.jit.metainterp.compile import CompileLoopVersionDescr
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref
@@ -600,12 +599,12 @@
         # at the end of self.mc.
         for tok in self.pending_guard_tokens:
             descr = tok.faildescr
-            if not isinstance(descr, CompileLoopVersionDescr):
+            if descr.loop_version():
+                startpos = self.mc.get_relative_pos()
+                self.store_info_on_descr(startpos, tok)
+            else:
                 regalloc.position = tok.position
                 tok.pos_recovery_stub = self.generate_quick_failure(tok, regalloc)
-            else:
-                startpos = self.mc.get_relative_pos()
-                self.store_info_on_descr(startpos, tok)
         if WORD == 8 and len(self.pending_memoryerror_trampoline_from) > 0:
             self.error_trampoline_64 = self.generate_propagate_error_64()
 
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -27,6 +27,7 @@
     FLOAT, VECTOR, TargetToken)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
 from rpython.jit.metainterp.compile import ResumeGuardDescr
+from rpython.jit.metainterp.resume import AccumInfo
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
@@ -320,37 +321,20 @@
                 continue
             accum = arg.getaccum()
             if accum:
+                # for an accumulator store the position of the original
+                # box and in llsupport/assembler save restore information
+                # on the descriptor
                 loc = self.loc(accum.getoriginalbox())
                 faillocs.append(loc)
-                self.update_accumulation_loc(arg, accum, descr, i)
+                descr.rd_accum_list = AccumInfo(descr.rd_accum_list,
+                                                i, accum.operator,
+                                                accum.getoriginalbox(),
+                                                self.loc(arg))
             else:
                 faillocs.append(self.loc(arg))
 
         return faillocs
 
-    def update_accumulation_loc(self, arg, accum, descr, pos):
-        """
-        Faillocs saved on the guard can only represent one value.
-        Accumulation has the accumulation box which need to updated uppon
-        guard exit. The fail descr saves where (regloc) the accumulator
-        is located.
-        """
-        assert isinstance(descr, ResumeGuardDescr)
-        accum_info = descr.rd_accum_list
-        count = 0
-        while accum_info:
-            if accum_info.box is accum.getoriginalbox():
-                accum_info.loc = self.loc(arg)
-                accum_info.position = pos
-                break
-            count += 1
-            accum_info = accum_info.prev
-        else:
-            msg = "[accumulator] %d accumulators, none matched box %s" % (count, accum_info.box)
-            print msg
-            import pdb; pdb.set_trace()
-            not_implemented(msg)
-
     def perform_with_guard(self, op, guard_op, arglocs, result_loc):
         faillocs = self.locs_for_fail(guard_op)
         self.rm.position += 1
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -84,19 +84,19 @@
         assert regalloc is not None
         accum_info = faildescr.rd_accum_list
         while accum_info:
-            pos = accum_info.position
-            loc = accum_info.loc
-            tgtloc = fail_locs[pos]
+            pos = accum_info.scalar_position
+            scalar_loc = fail_locs[pos]
+            vector_loc = accum_info.vector_loc
             # the upper elements will be lost if saved to the stack!
-            assert isinstance(loc, RegLoc)
-            if not isinstance(tgtloc, RegLoc):
-                tgtloc = regalloc.force_allocate_reg(accum_info.box)
-            arg = accum_info.box
-            assert arg is not None
-            if accum_info.operation == '+':
-                self._accum_reduce_sum(arg, loc, tgtloc)
-            elif accum_info.operation == '*':
-                self._accum_reduce_mul(arg, loc, tgtloc)
+            scalar_arg = accum_info.scalar_box
+            assert isinstance(vector_loc, RegLoc)
+            if not isinstance(scalar_loc, RegLoc):
+                scalar_loc = regalloc.force_allocate_reg(scalar_arg)
+            assert scalar_arg is not None
+            if accum_info.accum_operation == '+':
+                self._accum_reduce_sum(scalar_arg, vector_loc, scalar_loc)
+            elif accum_info.accum_operation == '*':
+                self._accum_reduce_mul(scalar_arg, vector_loc, scalar_loc)
             else:
                 not_implemented("accum operator %s not implemented" %
                                             (accum_info.operation)) 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -550,13 +550,6 @@
     def accumulates_value(self):
         return True
 
-    def save_to_descr(self, descr, position):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        from rpython.jit.metainterp.resume import AccumInfo
-        assert isinstance(descr, ResumeGuardDescr)
-        ai = AccumInfo(descr.rd_accum_list, position, self.operator, self.var)
-        descr.rd_accum_list = ai
-
 class BoxVector(Box):
     type = VECTOR
     _attrs_ = ('item_type','item_count','item_size','item_signed','accum')
@@ -819,7 +812,7 @@
         token = TargetToken(jitcell_token)
         token.original_jitcell_token = jitcell_token
         all_target_tokens.append(token)
-        if label.getdescr() is not jump.getdescr():
+        if label.getdescr() is None or label.getdescr() is not jump.getdescr():
             label_index = index_of_first(rop.LABEL, self.operations, 1)
             if label_index > 0:
                 second_label = self.operations[label_index]
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -213,6 +213,7 @@
         if len(others) > 0: # (2)
             replaced = False
             for i,other in enumerate(others):
+                assert guard is not other
                 if guard.implies(other, self):
                     # strengthend
                     others[i] = guard
@@ -223,6 +224,7 @@
                     continue
                 elif other.implies(guard, self):
                     # implied
+                    guard.rd_accum_list = None
                     self.guards[guard.index] = None # mark as 'do not emit'
                     replaced = True
                     continue
@@ -300,6 +302,7 @@
             for other in guards[1:]:
                 transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
+                    transitive_guard.rd_accum_list = None
                     other.set_to_none(loop.operations)
                     root_version.register_guard(transitive_guard, version)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1412,8 +1412,19 @@
         guard_false(i33, descr=) [p1, p0, p6, p7, i29, None, None]
         jump(p0, p1, p6, p7, i29, p14, p15)
         """        
-        #opt = self.schedule(self.parse_loop(trace))
-        #self.debug_print_operations(opt.loop)
+        trace = """
+        [p0, p1, p6, p7, p8, p11, p13, p15, i46, f43, i32, i36, i40]
+        i51 = int_lt(i46, i32)
+        guard_true(i51, descr=) [p1, p0, p15, p6, p7, p8, p11, p13, f43, i46]
+        i52 = int_lt(i46, i36)
+        guard_true(i52, descr=) [p1, p0, p11, i46, p6, p7, p8, p13, p15, f43, None]
+        f54 = getarrayitem_raw(i40, i46, descr=floatarraydescr)
+        f55 = float_add(f43, f54)
+        i56 = int_add(i46, 1)
+        jump(p0, p1, p6, p7, p8, p11, p13, p15, i56, f55, i32, i36, i40)
+        """
+        opt = self.schedule(self.parse_loop(trace))
+        self.debug_print_operations(opt.loop)
 
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -509,15 +509,16 @@
                 # this needs to be done for renamed (accum arguments)
                 version.renamed_inputargs = [ renamer.rename_map.get(arg,arg) for arg in version.inputargs ]
             self.appended_arg_count = len(sched_data.invariant_vector_vars)
-            for guard_node in self.dependency_graph.guards:
-                op = guard_node.getoperation()
-                failargs = op.getfailargs()
-                for i,arg in enumerate(failargs):
-                    if arg is None:
-                        continue
-                    accum = arg.getaccum()
-                    if accum:
-                        accum.save_to_descr(op.getdescr(),i)
+            #for guard_node in self.dependency_graph.guards:
+            #    op = guard_node.getoperation()
+            #    failargs = op.getfailargs()
+            #    for i,arg in enumerate(failargs):
+            #        if arg is None:
+            #            continue
+            #        accum = arg.getaccum()
+            #        if accum:
+            #            pass
+            #            #accum.save_to_descr(op.getdescr(),i)
             self.has_two_labels = len(sched_data.invariant_oplist) > 0
             self.loop.operations = self.prepend_invariant_operations(sched_data)
         else:
diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py
--- a/rpython/jit/metainterp/resume.py
+++ b/rpython/jit/metainterp/resume.py
@@ -35,13 +35,20 @@
         self.pc = pc
 
 class AccumInfo(object):
-    __slots__ = ('prev', 'position', 'operation', 'box', 'loc')
-    def __init__(self, prev, position, operation, box):
+    __slots__ = ('prev', 'accum_operation', 'scalar_position', 'scalar_box', 'vector_loc')
+    def __init__(self, prev, position, operation, box, loc):
         self.prev = prev
-        self.operation = operation
-        self.position = position
-        self.box = box
-        self.loc = None
+        self.accum_operation = operation
+        self.scalar_position = position
+        self.scalar_box = box
+        self.vector_loc = loc
+
+    def __repr__(self):
+        return 'AccumInfo(%s,%s,%s,%s,%s)' % (self.prev is None,
+                                              self.accum_operation,
+                                              self.scalar_position,
+                                              self.scalar_box,
+                                              self.vector_loc)
 
 def _ensure_parent_resumedata(framestack, n):
     target = framestack[n]
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -220,8 +220,9 @@
         res = self.meta_interp(f, [60,58.4547])
         assert res == f(60,58.4547) == 58.4547
 
-    def test_accum(self):
-        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=True)
+    @py.test.mark.parametrize('vec,vec_all',[(False,True),(True,False),(True,True),(False,False)])
+    def test_accum(self, vec, vec_all):
+        myjitdriver = JitDriver(greens = [], reds = 'auto', vectorize=vec)
         T = lltype.Array(rffi.DOUBLE)
         def f(d, value):
             va = lltype.malloc(T, d, flavor='raw', zero=True)
@@ -239,7 +240,7 @@
                 i += 1
             lltype.free(va, flavor='raw')
             return r
-        res = self.meta_interp(f, [60,0.5], vec=True)
+        res = self.meta_interp(f, [60,0.5], vec=vec, vec_all=vec_all)
         assert res == f(60,0.5) == 60*0.5
 
 

From noreply at buildbot.pypy.org  Sun Aug 23 17:16:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 17:16:04 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: small fixes
Message-ID: <20150823151604.2941A1C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79157:c3bc8ad8a836
Date: 2015-08-23 17:16 +0200
http://bitbucket.org/pypy/pypy/changeset/c3bc8ad8a836/

Log:	small fixes

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -167,7 +167,8 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, infos, self.optimizer.cpu)
+                    virtual_state, jump_op.getarglist(), infos,
+                    self.optimizer.cpu)
                 assert not extra_guards.extra_guards
             except VirtualStatesCantMatch:
                 continue
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -347,7 +347,8 @@
 
         extra_guards = state.extra_guards
         cpu = state.cpu
-        if self.lenbound and not self.lenbound.contains_bound(other.lenbound):
+        if (self.lenbound and other.lenbound and
+                not self.lenbound.contains_bound(other.lenbound)):
             raise VirtualStatesCantMatch("length bound does not match")
 
         if self.level == LEVEL_UNKNOWN:
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -1428,7 +1428,7 @@
         res = self.meta_interp(f, [6, 7])
         assert res == 42
         self.check_trace_count(1)
-        self.check_resops(call=2)
+        self.check_resops(call_r=2)
 
     def test_merge_guardclass_guardvalue(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'l'])

From noreply at buildbot.pypy.org  Sun Aug 23 17:21:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 17:21:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix more tests
Message-ID: <20150823152147.5D4491C137E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79158:ed3d726ab106
Date: 2015-08-23 17:22 +0200
http://bitbucket.org/pypy/pypy/changeset/ed3d726ab106/

Log:	fix more tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -116,3 +116,6 @@
         jump(1, 3)
         """
         self.optimize(loop, bridge, expected)
+
+    def test_virtual_state_guard_needed(self):
+        pass
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -7,7 +7,7 @@
      Optimization, LoopInfo, MININT, MAXINT
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp import compile
 
 class UnrollableOptimizer(Optimizer):
@@ -169,7 +169,13 @@
                 extra_guards = target_virtual_state.generate_guards(
                     virtual_state, jump_op.getarglist(), infos,
                     self.optimizer.cpu)
-                assert not extra_guards.extra_guards
+                patchguardop = self.optimizer.patchguardop
+                for guard in extra_guards.extra_guards:
+                    if isinstance(guard, GuardResOp):
+                        guard.rd_snapshot = patchguardop.rd_snapshot
+                        guard.rd_frame_info_list = patchguardop.rd_frame_info_list
+                        guard.setdescr(compile.ResumeAtPositionDescr())
+                    self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
             short_preamble = target_token.short_preamble
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -1606,7 +1606,7 @@
                 y.v = g(y.v) - y.v/y.v + lc/l[0] - 1
             return y.v
         res = self.meta_interp(f, [20], listops=True)
-        self.check_resops(getarrayitem_gc=0, getfield_gc=1)
+        self.check_resops(getarrayitem_gc_i=0, getfield_gc_i=1)
 
     def test_guard_isnull_nonnull(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'res'])
@@ -1661,7 +1661,7 @@
             return res
         res = self.meta_interp(g, [21])
         assert res == 3 * 21
-        self.check_resops(call=1)
+        self.check_resops(call_r=1)
 
     def test_bug_optimizeopt_mutates_ops(self):
         myjitdriver = JitDriver(greens = [], reds = ['x', 'res', 'const', 'a'])
@@ -1823,7 +1823,7 @@
         assert res == 6*8 + 6**8
         self.check_trace_count(4)
         self.check_resops({'guard_class': 2, 'int_gt': 4,
-                           'getfield_gc': 4, 'guard_true': 4,
+                           'getfield_gc_i': 4, 'guard_true': 4,
                            'int_sub': 4, 'jump': 2, 'int_mul': 2,
                            'int_add': 2})
 
@@ -1866,7 +1866,7 @@
         res = self.meta_interp(g, [6, 20])
         assert res == g(6, 20)
         self.check_trace_count(8)
-        self.check_resops(getarrayitem_gc=10)
+        self.check_resops(getarrayitem_gc_i=10)
 
     def test_multiple_specialied_versions_bridge(self):
         myjitdriver = JitDriver(greens = [], reds = ['y', 'x', 'z', 'res'])

From noreply at buildbot.pypy.org  Sun Aug 23 17:24:05 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 17:24:05 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: a all new stitch bridge that
 considers register mapping. works for accumulation values as well
Message-ID: <20150823152405.CE8451C137E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79159:0a4078644343
Date: 2015-08-23 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/0a4078644343/

Log:	a all new stitch bridge that considers register mapping. works for
	accumulation values as well

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -590,9 +590,57 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
 
-    def stitch_bridge(self, faildescr, target):
-        assert target.rawstart != 0
-        self.patch_jump_for_descr(faildescr, target.rawstart)
+    def stitch_bridge(self, faildescr, version):
+        """ Stitching means that one can enter a bridge with a complete different register
+            allocation. This needs remapping which is done here for both normal registers
+            and accumulation registers.
+            Why? Because this only generates a very small junk of memory, instead of
+            duplicating the loop assembler!
+        """
+        asminfo, bridge_faildescr, compiled_version, looptoken = version._compiled
+        assert asminfo.rawstart != 0
+        self.mc = codebuf.MachineCodeBlockWrapper()
+        allblocks = self.get_asmmemmgr_blocks(looptoken)
+        self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr,
+                                                   allblocks)
+        frame_info = self.datablockwrapper.malloc_aligned(
+            jitframe.JITFRAMEINFO_SIZE, alignment=WORD)
+
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
+        # if accumulation is saved at the guard, we need to update it here!
+        guard_locs = self.rebuild_faillocs_from_descr(faildescr, version.inputargs)
+        bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, compiled_version.inputargs)
+        guard_accum_info = faildescr.rd_accum_list
+        # O(n^2), but usually you only have at most 1 fail argument
+        while guard_accum_info:
+            bridge_accum_info = bridge_faildescr.rd_accum_list
+            while bridge_accum_info:
+                if bridge_accum_info.scalar_position == guard_accum_info.scalar_position:
+                    # the mapping might be wrong!
+                    if bridge_accum_info.vector_loc is not guard_accum_info.vector_loc:
+                        self.mov(guard_accum_info.vector_loc, bridge_accum_info.vector_loc)
+                bridge_accum_info = bridge_accum_info.prev
+            guard_accum_info = guard_accum_info.prev
+
+        # register mapping is most likely NOT valid, thus remap it in this
+        # short piece of assembler
+        assert len(guard_locs) == len(bridge_locs)
+        for i,gloc in enumerate(guard_locs):
+            bloc = bridge_locs[i]
+            bstack = bloc.location_code() == 'b'
+            gstack = gloc.location_code() == 'b'
+            if bstack and gstack:
+                pass
+            elif gloc is not bloc:
+                self.mov(gloc, bloc)
+        self.mc.JMP_l(0)
+        self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
+        offset = self.mc.get_relative_pos() - 4
+        rawstart = self.materialize_loop(looptoken)
+        # update the exit target
+        self._patch_jump_for_descr(rawstart + offset, asminfo.rawstart)
+        # update the guard to jump right to this custom piece of assembler
+        self.patch_jump_for_descr(faildescr, rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
@@ -732,6 +780,10 @@
 
     def patch_jump_for_descr(self, faildescr, adr_new_target):
         adr_jump_offset = faildescr.adr_jump_offset
+        self._patch_jump_for_descr(adr_jump_offset, adr_new_target)
+        faildescr.adr_jump_offset = 0    # means "patched"
+
+    def _patch_jump_for_descr(self, adr_jump_offset, adr_new_target):
         assert adr_jump_offset != 0
         offset = adr_new_target - (adr_jump_offset + 4)
         # If the new target fits within a rel32 of the jump, just patch
@@ -752,7 +804,6 @@
             p = rffi.cast(rffi.INTP, adr_jump_offset)
             adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0])
             mc.copy_to_raw_memory(adr_target)
-        faildescr.adr_jump_offset = 0    # means "patched"
 
     def fixup_target_tokens(self, rawstart):
         for targettoken in self.target_tokens_currently_compiling:
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -209,9 +209,8 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version._compiled = asminfo
+            version._compiled = (asminfo, faildescr, faildescr.version, jitcell_token)
             faildescr.version = None
-        # stitch the rest of the traces
         for lv in loop.versions:
             if not lv.compiled():
                 # the version was never compiled, do not bother
@@ -221,7 +220,7 @@
                 assert isinstance(faildescr, CompileLoopVersionDescr)
                 version = faildescr.version
                 if version and version.compiled():
-                    cpu.stitch_bridge(faildescr, version._compiled)
+                    cpu.stitch_bridge(faildescr, version)
                 faildescr.version = None
     loop.versions = None
 

From noreply at buildbot.pypy.org  Sun Aug 23 17:49:23 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 17:49:23 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: pass arraydescr a bit
 everywhere for short preamble
Message-ID: <20150823154923.5CCDF1C118C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79160:ed85c661a0e3
Date: 2015-08-23 17:49 +0200
http://bitbucket.org/pypy/pypy/changeset/ed85c661a0e3/

Log:	pass arraydescr a bit everywhere for short preamble

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -198,17 +198,17 @@
         return op.getarg(2)
 
     def _getfield(self, opinfo, descr, optheap):
-        res = opinfo.getitem(self.index, optheap)
+        res = opinfo.getitem(descr, self.index, optheap)
         if isinstance(res, PreambleOp):
             index = res.preamble_op.getarg(1).getint()
             res = optheap.optimizer.force_op_from_preamble(res)
-            opinfo.setitem(index, None, res, optheap=optheap)
+            opinfo.setitem(descr, index, None, res, optheap=optheap)
         return res
 
     def _setfield(self, op, opinfo, optheap):
         arg = optheap.get_box_replacement(op.getarg(2))
         struct = optheap.get_box_replacement(op.getarg(0))
-        opinfo.setitem(self.index, struct, arg, self, optheap)
+        opinfo.setitem(op.getdescr(), self.index, struct, arg, self, optheap)
 
     def invalidate(self, descr):
         for opinfo in self.cached_infos:
@@ -589,7 +589,7 @@
         self.emit_operation(op)
         # the remember the result of reading the array item
         if cf is not None:
-            arrayinfo.setitem(indexb.getint(),
+            arrayinfo.setitem(op.getdescr(), indexb.getint(),
                               self.get_box_replacement(op.getarg(0)),
                               self.get_box_replacement(op), cf,
                               self)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -317,16 +317,19 @@
         return visitor.visit_vrawslice(self.offset)
 
 class ArrayPtrInfo(AbstractVirtualPtrInfo):
-    _attrs_ = ('length', '_items', 'lenbound', '_clear')
+    _attrs_ = ('length', '_items', 'lenbound', '_clear', 'arraydescr')
 
     _items = None
     lenbound = None
     length = -1
 
-    def __init__(self, const=None, size=0, clear=False, vdescr=None):
+    def __init__(self, arraydescr, const=None, size=0, clear=False,
+                 vdescr=None):
         from rpython.jit.metainterp.optimizeopt import intutils
         self.vdescr = vdescr
+        self.arraydescr = arraydescr
         if vdescr is not None:
+            assert vdescr is arraydescr
             self._init_items(const, size, clear)
             self.lenbound = intutils.ConstIntBound(size)
         self._clear = clear
@@ -350,9 +353,9 @@
     def all_items(self):
         return self._items
 
-    def copy_fields_to_const(self, constinfo, optheap):
+    def copy_fields_to_const(self, arraydescr, constinfo, optheap):
         if self._items is not None:
-            info = constinfo._get_array_info(optheap)
+            info = constinfo._get_array_info(arraydescr, optheap)
             info._items = self._items[:]
 
     def _force_elements(self, op, optforce, descr):
@@ -371,7 +374,7 @@
                     optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
-    def setitem(self, index, struct_op, op, cf=None, optheap=None):
+    def setitem(self, descr, index, struct_op, op, cf=None, optheap=None):
         if self._items is None:
             self._items = [None] * (index + 1)
         if index >= len(self._items):
@@ -381,7 +384,7 @@
             assert not self.is_virtual()
             cf.register_dirty_field(struct_op, self)
 
-    def getitem(self, index, optheap=None):
+    def getitem(self, descr, index, optheap=None):
         if self._items is None or index >= len(self._items):
             return None
         return self._items[index]
@@ -418,7 +421,7 @@
     def make_guards(self, op, short):
         AbstractVirtualPtrInfo.make_guards(self, op, short)
         if self.lenbound is not None:
-            lenop = ResOperation(rop.ARRAYLEN_GC, [op])
+            lenop = ResOperation(rop.ARRAYLEN_GC, [op], descr=self.arraydescr)
             short.append(lenop)
             self.lenbound.make_guards(lenop, short)
 
@@ -503,11 +506,11 @@
             optheap.const_infos[ref] = info
         return info
 
-    def _get_array_info(self, optheap):
+    def _get_array_info(self, arraydescr, optheap):
         ref = self._const.getref_base()
         info = optheap.const_infos.get(ref, None)
         if info is None:
-            info = ArrayPtrInfo()
+            info = ArrayPtrInfo(arraydescr)
             optheap.const_infos[ref] = info
         return info        
 
@@ -515,13 +518,13 @@
         info = self._get_info(optheap)
         return info.getfield(descr)
 
-    def getitem(self, index, optheap=None):
-        info = self._get_array_info(optheap)
-        return info.getitem(index)
+    def getitem(self, descr, index, optheap=None):
+        info = self._get_array_info(descr, optheap)
+        return info.getitem(descr, index)
 
-    def setitem(self, index, struct, op, cf=None, optheap=None):
-        info = self._get_array_info(optheap)
-        info.setitem(index, struct, op, cf)
+    def setitem(self, descr, index, struct, op, cf=None, optheap=None):
+        info = self._get_array_info(descr, optheap)
+        info.setitem(descr, index, struct, op, cf)
 
     def setfield(self, descr, struct, op, optheap=None, cf=None):
         info = self._get_info(optheap)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -549,7 +549,7 @@
             # XXX fish fish fish
             for index in range(length.getint()):
                 if source_info and source_info.is_virtual():
-                    val = source_info.getitem(index + source_start)
+                    val = source_info.getitem(arraydescr, index + source_start)
                 else:
                     opnum = OpHelpers.getarrayitem_for_descr(arraydescr)
                     newop = ResOperation(opnum,
@@ -561,7 +561,7 @@
                 if val is None:
                     continue
                 if dest_info and dest_info.is_virtual():
-                    dest_info.setitem(index + dest_start,
+                    dest_info.setitem(arraydescr, index + dest_start,
                                       self.get_box_replacement(op.getarg(2)),
                                       val)
                 else:
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -69,7 +69,8 @@
         else:
             index = g.getarg(1).getint()
             assert index >= 0
-            opinfo.setitem(index, self.res, pop, optheap=optheap)
+            opinfo.setitem(self.getfield_op.getdescr(), index, self.res,
+                           pop, optheap=optheap)
 
     def add_op_to_short(self, sb):
         sop = self.getfield_op
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_virtualstate.py
@@ -489,7 +489,7 @@
         info2 = VArrayStateInfo(descr)
         info2.fieldstate = [innerinfo2]
 
-        value1 = info.ArrayPtrInfo(vdescr=descr, size=1)
+        value1 = info.ArrayPtrInfo(descr, vdescr=descr, size=1)
         box = InputArgRef(self.nodeaddr)
         value1._items[0] = box
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -48,7 +48,7 @@
             if known_class:
                 self.make_constant_class(op, known_class, False)
             if isinstance(preamble_info, info.ArrayPtrInfo):
-                arr_info = info.ArrayPtrInfo(None)
+                arr_info = info.ArrayPtrInfo(preamble_info.arraydescr)
                 arr_info.lenbound = preamble_info.getlenbound(None)
                 op.set_forwarded(arr_info)
             if preamble_info.is_nonnull():
@@ -210,25 +210,27 @@
 
     def inline_short_preamble(self, jump_args, short_inputargs, short_ops,
                               short_jump_op, patchguardop):
-        self._check_no_forwarding([short_inputargs, short_ops], False)
-        assert len(short_inputargs) == len(jump_args)
-        for i in range(len(jump_args)):
-            short_inputargs[i].set_forwarded(None)
-            self.make_equal_to(short_inputargs[i], jump_args[i])
-        for op in short_ops:
-            if op.is_guard():
-                op = self.replace_op_with(op, op.getopnum(),
-                                          descr=compile.ResumeAtPositionDescr())
-                op.rd_snapshot = patchguardop.rd_snapshot
-                op.rd_frame_info_list = patchguardop.rd_frame_info_list
-            self.optimizer.send_extra_operation(op)
-        res = [self.optimizer.get_box_replacement(op) for op in
-                short_jump_op]
-        for op in short_inputargs:
-            op.set_forwarded(None)
-        for op in short_ops:
-            op.set_forwarded(None)
-        return res
+        try:
+            self._check_no_forwarding([short_inputargs, short_ops], False)
+            assert len(short_inputargs) == len(jump_args)
+            for i in range(len(jump_args)):
+                short_inputargs[i].set_forwarded(None)
+                self.make_equal_to(short_inputargs[i], jump_args[i])
+            for op in short_ops:
+                if op.is_guard():
+                    op = self.replace_op_with(op, op.getopnum(),
+                                    descr=compile.ResumeAtPositionDescr())
+                    op.rd_snapshot = patchguardop.rd_snapshot
+                    op.rd_frame_info_list = patchguardop.rd_frame_info_list
+                self.optimizer.send_extra_operation(op)
+            res = [self.optimizer.get_box_replacement(op) for op in
+                    short_jump_op]
+            return res
+        finally:
+            for op in short_inputargs:
+                op.set_forwarded(None)
+            for op in short_ops:
+                op.set_forwarded(None)
 
     def export_state(self, start_label, original_label_args, renamed_inputargs):
         end_args = [self.get_box_replacement(a) for a in original_label_args]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -28,7 +28,8 @@
             opinfo = info.ArrayStructInfo(size, vdescr=arraydescr)
         else:
             const = self.new_const_item(arraydescr)
-            opinfo = info.ArrayPtrInfo(const, size, clear, vdescr=arraydescr)
+            opinfo = info.ArrayPtrInfo(arraydescr, const, size, clear,
+                                       vdescr=arraydescr)
         newop = self.replace_op_with(source_op, source_op.getopnum())
         newop.set_forwarded(opinfo)
         return opinfo
@@ -277,7 +278,7 @@
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
-                item = opinfo.getitem(indexbox.getint())
+                item = opinfo.getitem(op.getdescr(), indexbox.getint())
                 if item is None:   # reading uninitialized array items?
                     assert False, "can't read uninitialized items"
                     itemvalue = value.constvalue     # bah, just return 0
@@ -299,7 +300,7 @@
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
-                opinfo.setitem(indexbox.getint(),
+                opinfo.setitem(op.getdescr(), indexbox.getint(),
                                self.get_box_replacement(op.getarg(0)),
                                self.get_box_replacement(op.getarg(2)))
                 return
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -223,7 +223,7 @@
         if len(self.fieldstate) > info.getlength():
             raise BadVirtualState
         for i in range(len(self.fieldstate)):
-            fieldbox = info.getitem(i)
+            fieldbox = info.getitem(self.arraydescr, i)
             if fieldbox is None:
                 xxx
                 v = value.get_missing_null_value()
diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py
--- a/rpython/jit/metainterp/test/test_resume.py
+++ b/rpython/jit/metainterp/test/test_resume.py
@@ -1285,7 +1285,8 @@
     modifier.liveboxes = {}
     modifier.vfieldboxes = {}
 
-    v2 = info.ArrayPtrInfo(vdescr=LLtypeMixin.arraydescr, size=2)
+    v2 = info.ArrayPtrInfo(LLtypeMixin.arraydescr,
+                           vdescr=LLtypeMixin.arraydescr, size=2)
     b2s.set_forwarded(v2)
     v2._items = [b4s, c1s]
     modifier.register_virtual_fields(b2s, [b4s, c1s])

From noreply at buildbot.pypy.org  Sun Aug 23 17:50:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 17:50:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix the test
Message-ID: <20150823155026.28FC01C118C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79161:83797180a4da
Date: 2015-08-23 17:50 +0200
http://bitbucket.org/pypy/pypy/changeset/83797180a4da/

Log:	fix the test

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2505,7 +2505,7 @@
                 if counter > 10:
                     return 7
         assert self.meta_interp(build, []) == 7
-        self.check_resops(getfield_gc_pure=2)
+        self.check_resops(getfield_gc_pure_r=2)
 
     def test_args_becomming_equal(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'i', 'sa', 'a', 'b'])

From noreply at buildbot.pypy.org  Sun Aug 23 18:23:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 18:23:50 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: don't propagate big intbounds
	in virtualstate
Message-ID: <20150823162350.31EAB1C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79162:1b0cc8297a84
Date: 2015-08-23 17:57 +0200
http://bitbucket.org/pypy/pypy/changeset/1b0cc8297a84/

Log:	don't propagate big intbounds in virtualstate

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -3,7 +3,7 @@
         ConstPtr, ConstFloat)
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded,\
-     ConstIntBound
+     ConstIntBound, MININT, MAXINT
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
@@ -325,6 +325,10 @@
                 # XXX strings?
                 self.lenbound = info.getlenbound(None)
         elif type == 'i':
+            if info.lower < MININT / 2:
+                info.lower = MININT
+            if info.upper > MAXINT / 2:
+                info.upper = MAXINT
             self.intbound = info
 
     def is_const(self):

From noreply at buildbot.pypy.org  Sun Aug 23 18:23:52 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 18:23:52 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some more tests
Message-ID: <20150823162352.652BA1C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79163:a747291d1194
Date: 2015-08-23 18:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a747291d1194/

Log:	fix some more tests

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,4 +1,5 @@
 
+import sys
 from rpython.jit.metainterp.history import Const
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
      ShortPreambleBuilder, PreambleOp
@@ -9,6 +10,7 @@
     VirtualStateConstructor, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
 from rpython.jit.metainterp import compile
+from rpython.rlib.debug import debug_print
 
 class UnrollableOptimizer(Optimizer):
     def force_op_from_preamble(self, preamble_op):
@@ -142,20 +144,41 @@
         jump_op = operations[-1]
         cell_token = jump_op.getdescr()
         if not inline_short_preamble or len(cell_token.target_tokens) == 1:
-            assert cell_token.target_tokens[0].virtual_state is None
-            jump_op = jump_op.copy_and_change(rop.JUMP,
-                                        descr=cell_token.target_tokens[0])
-            self.optimizer.send_extra_operation(jump_op)
-            return info, self.optimizer._newoperations[:]
+            return self.jump_to_preamble(cell_token, jump_op, info)
         vs = self.jump_to_existing_trace(jump_op, inline_short_preamble)
         if vs is None:
             return info, self.optimizer._newoperations[:]
+        warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc
+        limit = warmrunnerdescr.memory_manager.retrace_limit
+        if cell_token.retraced_count < limit:
+            cell_token.retraced_count += 1
+            debug_print('Retracing (%d/%d)' % (cell_token.retraced_count, limit))
+        else:
+            debug_print("Retrace count reached, jumping to preamble")
+            return self.jump_to_preamble(cell_token, jump_op, info)
+        maxguards = warmrunnerdescr.memory_manager.max_retrace_guards
+        guard_count = 0
+        for op in self.optimizer._newoperations:
+            if op.is_guard():
+                guard_count += 1
+        if guard_count > maxguards:
+            target_token = cell_token.target_tokens[0]
+            target_token.targeting_jitcell_token.retraced_count = sys.maxint
+            return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(start_label,
                                            operations[-1].getarglist(),
                                            info.inputargs)
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
+    def jump_to_preamble(self, cell_token, jump_op, info):
+        assert cell_token.target_tokens[0].virtual_state is None
+        jump_op = jump_op.copy_and_change(rop.JUMP,
+                                          descr=cell_token.target_tokens[0])
+        self.optimizer.send_extra_operation(jump_op)
+        return info, self.optimizer._newoperations[:]
+
+
     def jump_to_existing_trace(self, jump_op, inline_short_preamble):
         jitcelltoken = jump_op.getdescr()
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -2,8 +2,8 @@
 from rpython.jit.metainterp.history import (ConstInt, Const,
         ConstPtr, ConstFloat)
 from rpython.jit.metainterp.optimizeopt import info
-from rpython.jit.metainterp.optimizeopt.intutils import IntUnbounded,\
-     ConstIntBound, MININT, MAXINT
+from rpython.jit.metainterp.optimizeopt.intutils import \
+     ConstIntBound, MININT, MAXINT, IntBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      AbstractInputArg
 from rpython.rlib.debug import debug_start, debug_stop, debug_print
@@ -325,11 +325,12 @@
                 # XXX strings?
                 self.lenbound = info.getlenbound(None)
         elif type == 'i':
-            if info.lower < MININT / 2:
-                info.lower = MININT
-            if info.upper > MAXINT / 2:
-                info.upper = MAXINT
-            self.intbound = info
+            if isinstance(info, IntBound):
+                if info.lower < MININT / 2:
+                    info.lower = MININT
+                if info.upper > MAXINT / 2:
+                    info.upper = MAXINT
+                self.intbound = info
 
     def is_const(self):
         return self.constbox is not None
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2767,9 +2767,8 @@
             return i
         #
         seen = []
-        def my_optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts,
-                              *args, **kwds):
-            seen.append('unroll' in enable_opts)
+        def my_optimize_trace(metainterp_sd, jitdriver_sd, data):
+            seen.append('unroll' in data.enable_opts)
             raise InvalidLoop
         old_optimize_trace = optimizeopt.optimize_trace
         optimizeopt.optimize_trace = my_optimize_trace
@@ -3011,7 +3010,7 @@
             return a[0].intvalue
         res = self.meta_interp(f, [100])
         assert res == -2
-        self.check_resops(setarrayitem_gc=2, getarrayitem_gc=1)
+        self.check_resops(setarrayitem_gc=2, getarrayitem_gc_r=1)
 
     def test_continue_tracing_with_boxes_in_start_snapshot_replaced_by_optimizer(self):
         myjitdriver = JitDriver(greens = [], reds = ['sa', 'n', 'a', 'b'])
@@ -3307,7 +3306,7 @@
                 lock.release()
             return n
         res = self.meta_interp(f, [10, 1])
-        self.check_resops(getfield_gc=4)
+        self.check_resops(getfield_gc_i=4)
         assert res == f(10, 1)
 
     def test_jit_merge_point_with_raw_pointer(self):
@@ -3371,10 +3370,10 @@
 
         res = self.meta_interp(main, [0, 10, 2], enable_opts='')
         assert res == main(0, 10, 2)
-        self.check_resops(call=1)
+        self.check_resops(call_i=1)
         res = self.meta_interp(main, [1, 10, 2], enable_opts='')
         assert res == main(1, 10, 2)
-        self.check_resops(call=0)
+        self.check_resops(call_i=0)
 
     def test_look_inside_iff_const_float(self):
         @look_inside_iff(lambda arg: isconstant(arg))
@@ -3393,7 +3392,7 @@
 
         res = self.meta_interp(main, [10], enable_opts='')
         assert res == 5.0
-        self.check_resops(call=1)
+        self.check_resops(call_f=1)
 
     def test_look_inside_iff_virtual(self):
         # There's no good reason for this to be look_inside_iff, but it's a test!
@@ -3418,10 +3417,10 @@
                     i += f(A(2), n)
         res = self.meta_interp(main, [0], enable_opts='')
         assert res == main(0)
-        self.check_resops(call=1, getfield_gc=0)
+        self.check_resops(call_i=1, getfield_gc_i=0)
         res = self.meta_interp(main, [1], enable_opts='')
         assert res == main(1)
-        self.check_resops(call=0, getfield_gc=0)
+        self.check_resops(call_i=0, getfield_gc_i=0)
 
     def test_isvirtual_call_assembler(self):
         driver = JitDriver(greens = ['code'], reds = ['n', 's'])

From noreply at buildbot.pypy.org  Sun Aug 23 18:54:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Aug 2015 18:54:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: Pass the test?
Message-ID: <20150823165447.A0FB31C0FF7@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79164:44cdf9256258
Date: 2015-08-23 18:55 +0200
http://bitbucket.org/pypy/pypy/changeset/44cdf9256258/

Log:	Pass the test?

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2723,17 +2723,23 @@
         """
         expected = """
         [p1, i2, i3]
-        guard_true(i3) [p1, i2]
+        guard_true(i3) [i2, p1]
         i4 = int_neg(i2)
         setfield_gc(p1, NULL, descr=nextdescr)
         jump(p1, i2, i4)
         """
         self.optimize_loop(ops, expected)
+        #
+        # initialize p1.getref_base() to return a random pointer to a NODE
+        # (it doesn't have to be self.nodeaddr, but it's convenient)
+        assert hasattr(self.oparse.getvar('p1'), '_resref')
+        self.oparse.getvar('p1')._resref = self.nodeaddr
+        #
         self.check_expanded_fail_descr(
             '''
             p1.nextdescr = p2
             where p2 is a node_vtable, valuedescr=i2
-            ''', rop.GUARD_TRUE, values=[InputArgInt(18),
+            ''', rop.GUARD_TRUE, values=[InputArgInt(0),
                                          InputArgRef(self.nodeaddr)])
 
     def test_expand_fail_lazy_setfield_2(self):

From noreply at buildbot.pypy.org  Sun Aug 23 19:40:30 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 19:40:30 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: extended the llgraph to handle the
 stitching correctly (not yet fully finished)
Message-ID: <20150823174030.E2B2F1C0FF7@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79165:c38afa242006
Date: 2015-08-23 19:40 +0200
http://bitbucket.org/pypy/pypy/changeset/c38afa242006/

Log:	extended the llgraph to handle the stitching correctly (not yet
	fully finished) translation issue

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -2,8 +2,8 @@
 from rpython.jit.backend import model
 from rpython.jit.backend.llgraph import support
 from rpython.jit.backend.llsupport import symbolic
-from rpython.jit.metainterp.history import AbstractDescr
-from rpython.jit.metainterp.history import Const, getkind
+from rpython.jit.metainterp.history import AbstractDescr, BoxVector
+from rpython.jit.metainterp.history import Const, getkind, Box
 from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID, VECTOR
 from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.optimizeopt import intbounds
@@ -271,6 +271,7 @@
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
     def stitch_bridge(self, faildescr, target):
+        import pdb; pdb.set_trace()
         faildescr._llgraph_bridge = target.lltrace
 
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
@@ -902,29 +903,36 @@
 
     # -----------------------------------------------------
 
+    def _accumulate(self, descr, failargs, value):
+        if not hasattr(descr, 'rd_accum_list'):
+            return
+        accum = descr.rd_accum_list
+        while accum is not None:
+            value = values[accum.scalar_position]
+            assert isinstance(value, list)
+            if accum.operation == '+':
+                value = sum(value)
+                break
+            elif accum.operation == '*':
+                def prod(acc, x): return acc * x
+                value = reduce(prod, value, 1)
+                break
+            else:
+                raise NotImplementedError("accum operator in fail guard")
+            values[accum.scalar_position] = value
+            accum = accum.prev
+
     def fail_guard(self, descr, saved_data=None):
         values = []
         for i,box in enumerate(self.current_op.getfailargs()):
+            if box.getaccum() or isinstance(box, BoxVector):
+                import pdb; pdb.set_trace();
             if box is not None:
                 value = self.env[box]
             else:
                 value = None
-            if hasattr(descr, 'rd_accum_list'):
-                accum = descr.rd_accum_list
-                while accum != None:
-                    if accum.position != i:
-                        accum = accum.prev
-                        continue
-                    if accum.operation == '+':
-                        value = sum(value)
-                        break
-                    elif accum.operation == '*':
-                        def prod(acc, x): return acc * x
-                        value = reduce(prod, value, 1)
-                        break
-                    else:
-                        raise NotImplementedError("accum operator in fail guard")
             values.append(value)
+        self._accumulate(descr, self.current_op.getfailargs(), values)
         if hasattr(descr, '_llgraph_bridge'):
             target = (descr._llgraph_bridge, -1)
             values = [value for value in values if value is not None]
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -326,6 +326,7 @@
                 # on the descriptor
                 loc = self.loc(accum.getoriginalbox())
                 faillocs.append(loc)
+                assert isinstance(descr, ResumeGuardDescr)
                 descr.rd_accum_list = AccumInfo(descr.rd_accum_list,
                                                 i, accum.operator,
                                                 accum.getoriginalbox(),

From noreply at buildbot.pypy.org  Sun Aug 23 20:27:04 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 23 Aug 2015 20:27:04 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix two tests
Message-ID: <20150823182704.7CB421C12F1@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79166:ccbfc9b9ffeb
Date: 2015-08-23 20:27 +0200
http://bitbucket.org/pypy/pypy/changeset/ccbfc9b9ffeb/

Log:	fix two tests

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3680,7 +3680,7 @@
             return n
         res = self.meta_interp(f, [10])
         assert res == 0
-        self.check_resops(call=0, call_may_force=0, new_array=0)
+        self.check_resops(call_i=0, call_may_force_i=0, new_array=0)
 
 
     def test_convert_from_SmallFunctionSetPBCRepr_to_FunctionsPBCRepr(self):
@@ -4101,7 +4101,9 @@
 
         res = self.meta_interp(f, [10])
         assert res == 2003     # two runs before jitting; then one tracing run
-        self.check_resops(int_add=0, call=0, call_may_force=0)
+        self.check_resops(int_add=0, call_i=0, call_may_force_i=0,
+                          call_r=0, call_may_force_r=0, call_f=0,
+                          call_may_force_f=0)
 
     def test_not_in_trace_exception(self):
         def g():

From noreply at buildbot.pypy.org  Sun Aug 23 20:31:23 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 20:31:23 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: llgraph working again with the new
 loop versioning (hurrayyyy!)
Message-ID: <20150823183123.3912E1C12F1@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79167:63513d022558
Date: 2015-08-23 20:31 +0200
http://bitbucket.org/pypy/pypy/changeset/63513d022558/

Log:	llgraph working again with the new loop versioning (hurrayyyy!)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -37,6 +37,8 @@
                 newbox = _cache[box]
             except KeyError:
                 newbox = _cache[box] = box.__class__()
+            if hasattr(box, 'accum') and box.accum:
+                newbox.accum = box.accum
             return newbox
         #
         self.inputargs = map(mapping, inputargs)
@@ -271,8 +273,7 @@
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
     def stitch_bridge(self, faildescr, target):
-        import pdb; pdb.set_trace()
-        faildescr._llgraph_bridge = target.lltrace
+        faildescr._llgraph_bridge = target._lltrace
 
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
                      unique_id=0, log=True, name='', logger=None):
@@ -291,6 +292,8 @@
         faildescr._llgraph_bridge = lltrace
         clt._llgraph_alltraces.append(lltrace)
         self._record_labels(lltrace)
+        if faildescr.loop_version():
+            faildescr.version._lltrace = lltrace
         return LLAsmInfo(lltrace)
 
     def _record_labels(self, lltrace):
@@ -903,30 +906,27 @@
 
     # -----------------------------------------------------
 
-    def _accumulate(self, descr, failargs, value):
-        if not hasattr(descr, 'rd_accum_list'):
-            return
-        accum = descr.rd_accum_list
-        while accum is not None:
-            value = values[accum.scalar_position]
+    def _accumulate(self, descr, failargs, values):
+        for i,box in enumerate(self.current_op.getfailargs()):
+            if box is None:
+                continue
+            accum = box.getaccum()
+            if not accum:
+                continue
+            value = values[i]
             assert isinstance(value, list)
-            if accum.operation == '+':
+            if accum.operator == '+':
                 value = sum(value)
-                break
-            elif accum.operation == '*':
+            elif accum.operator == '*':
                 def prod(acc, x): return acc * x
                 value = reduce(prod, value, 1)
-                break
             else:
                 raise NotImplementedError("accum operator in fail guard")
-            values[accum.scalar_position] = value
-            accum = accum.prev
+            values[i] = value
 
     def fail_guard(self, descr, saved_data=None):
         values = []
         for i,box in enumerate(self.current_op.getfailargs()):
-            if box.getaccum() or isinstance(box, BoxVector):
-                import pdb; pdb.set_trace();
             if box is not None:
                 value = self.env[box]
             else:
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -10,6 +10,7 @@
 from rpython.jit.metainterp.history import (Const, Box, VOID,
     BoxVector, ConstInt)
 from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT
+from rpython.jit.metainterp.compile import ResumeGuardDescr
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref
@@ -598,6 +599,8 @@
             duplicating the loop assembler!
         """
         asminfo, bridge_faildescr, compiled_version, looptoken = version._compiled
+        assert isinstance(bridge_faildescr, ResumeGuardDescr)
+        assert isinstance(faildescr, ResumeGuardDescr)
         assert asminfo.rawstart != 0
         self.mc = codebuf.MachineCodeBlockWrapper()
         allblocks = self.get_asmmemmgr_blocks(looptoken)
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -224,7 +224,6 @@
                     continue
                 elif other.implies(guard, self):
                     # implied
-                    guard.rd_accum_list = None
                     self.guards[guard.index] = None # mark as 'do not emit'
                     replaced = True
                     continue

From noreply at buildbot.pypy.org  Sun Aug 23 20:33:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Aug 2015 20:33:35 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: Fix the merge
Message-ID: <20150823183335.948D81C135C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r79168:fda6dfca8877
Date: 2015-08-23 20:33 +0200
http://bitbucket.org/pypy/pypy/changeset/fda6dfca8877/

Log:	Fix the merge

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -121,7 +121,8 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- GETFIELD_GC ----------
-            if op.getopnum() == rop.GETFIELD_GC:
+            if op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F,
+                                 rop.GETFIELD_GC_R):
                 self.handle_getfield_gc(op)
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
@@ -194,7 +195,7 @@
         followed by a bunch of 'setfields', and the 'pending_zeros'
         optimization we do here is meant for this case."""
         self.emit_pending_zeros()
-        self.newops.append(op)
+        self.emit_op(op)
 
     # ----------
 

From noreply at buildbot.pypy.org  Sun Aug 23 20:37:25 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 20:37:25 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: translation issue,
	assigning to wrong class
Message-ID: <20150823183725.4D61A1C137E@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79169:2e20dba31d03
Date: 2015-08-23 20:36 +0200
http://bitbucket.org/pypy/pypy/changeset/2e20dba31d03/

Log:	translation issue, assigning to wrong class

diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -301,7 +301,6 @@
             for other in guards[1:]:
                 transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
-                    transitive_guard.rd_accum_list = None
                     other.set_to_none(loop.operations)
                     root_version.register_guard(transitive_guard, version)
 

From noreply at buildbot.pypy.org  Sun Aug 23 20:51:08 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 20:51:08 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: added docu to LoopVersion,
 translation issue with tuple/None unification
Message-ID: <20150823185108.0FFF11C118C@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79170:70512ea6c862
Date: 2015-08-23 20:50 +0200
http://bitbucket.org/pypy/pypy/changeset/70512ea6c862/

Log:	added docu to LoopVersion, translation issue with tuple/None
	unification

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -596,9 +596,10 @@
             allocation. This needs remapping which is done here for both normal registers
             and accumulation registers.
             Why? Because this only generates a very small junk of memory, instead of
-            duplicating the loop assembler!
+            duplicating the loop assembler for each faildescr!
         """
-        asminfo, bridge_faildescr, compiled_version, looptoken = version._compiled
+        asminfo, bridge_faildescr, looptoken = version._compiled
+        compiled_version = bridge_faildescr.version
         assert isinstance(bridge_faildescr, ResumeGuardDescr)
         assert isinstance(faildescr, ResumeGuardDescr)
         assert asminfo.rawstart != 0
@@ -640,7 +641,7 @@
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         offset = self.mc.get_relative_pos() - 4
         rawstart = self.materialize_loop(looptoken)
-        # update the exit target
+        # update the jump to the real trace
         self._patch_jump_for_descr(rawstart + offset, asminfo.rawstart)
         # update the guard to jump right to this custom piece of assembler
         self.patch_jump_for_descr(faildescr, rawstart)
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -78,7 +78,8 @@
 
     def _accum_update_at_exit(self, fail_locs, fail_args, faildescr, regalloc):
         """ If accumulation is done in this loop, at the guard exit
-        some vector registers must be adjusted to yield the correct value"""
+            some vector registers must be adjusted to yield the correct value
+        """
         if not isinstance(faildescr, ResumeGuardDescr):
             return
         assert regalloc is not None
@@ -99,7 +100,7 @@
                 self._accum_reduce_mul(scalar_arg, vector_loc, scalar_loc)
             else:
                 not_implemented("accum operator %s not implemented" %
-                                            (accum_info.operation)) 
+                                            (accum_info.accum_operation)) 
             accum_info = accum_info.prev
 
     def _accum_reduce_mul(self, arg, accumloc, targetloc):
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -209,8 +209,9 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version._compiled = (asminfo, faildescr, faildescr.version, jitcell_token)
+            version._compiled = (asminfo, faildescr, jitcell_token)
             faildescr.version = None
+        # stitch to the trace loop
         for lv in loop.versions:
             if not lv.compiled():
                 # the version was never compiled, do not bother
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -740,10 +740,13 @@
     return -1
 
 class LoopVersion(object):
-
+    """ A special version of a trace loop. Use loop.snaphost() to
+        create one instance and attach it to a guard descr.
+        If not attached to a descriptor, it will not be compiled.
+    """
     def __init__(self, loop):
         self.faildescrs = []
-        self._compiled = None
+        self._compiled = (None,None,None)
         if loop:
             self.operations = self.copy_operations(loop.operations) 
             idx = index_of_first(rop.LABEL, self.operations)
@@ -761,7 +764,7 @@
             # root version must always be compiled
             return True
 
-        return self._compiled is not None
+        return self._compiled[0] is not None
 
     def copy_operations(self, operations):
         from rpython.jit.metainterp.compile import (ResumeGuardDescr,

From noreply at buildbot.pypy.org  Sun Aug 23 21:36:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Aug 2015 21:36:36 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: Translation fix
Message-ID: <20150823193636.ACD7E1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r79171:c3624e107b1b
Date: 2015-08-23 20:02 +0100
http://bitbucket.org/pypy/pypy/changeset/c3624e107b1b/

Log:	Translation fix

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -194,6 +194,11 @@
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
 
+    def _get_buffer(self):
+        buffer = self.buffer
+        assert buffer is not None
+        return buffer
+
     def getitem_raw(self, offset, itemsize, descr):
         if not self.is_virtual():
             raise InvalidRawOperation
@@ -201,37 +206,39 @@
             # for the test above: it's not enough to check is_virtual()
             # on the original object, because it might be a VRawSliceValue
             # instead.  If it is a virtual one, then we'll reach here anway.
-        return self.buffer.read_value(offset, itemsize, descr)
+        return self._get_buffer().read_value(offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         if not self.is_virtual():
             raise InvalidRawOperation
-        self.buffer.write_value(offset, itemsize, descr, itemop)
+        self._get_buffer().write_value(offset, itemsize, descr, itemop)
 
     def is_virtual(self):
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
         self.size = -1
-        for i in range(len(self.buffer.offsets)):
+        buffer = self._get_buffer()
+        for i in range(len(buffer.offsets)):
             # write the value
-            offset = self.buffer.offsets[i]
-            descr = self.buffer.descrs[i]
-            itembox = self.buffer.values[i]
+            offset = buffer.offsets[i]
+            descr = buffer.descrs[i]
+            itembox = buffer.values[i]
             op = ResOperation(rop.RAW_STORE,
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(op)
 
     def visitor_walk_recursive(self, op, visitor, optimizer):
-        itemboxes = self.buffer.values
+        itemboxes = self._get_buffer().values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
+        buffer = self._get_buffer()
         return visitor.visit_vrawbuffer(self.size,
-                                        self.buffer.offsets[:],
-                                        self.buffer.descrs[:])
+                                        buffer.offsets[:],
+                                        buffer.descrs[:])
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):

From noreply at buildbot.pypy.org  Sun Aug 23 21:36:39 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Aug 2015 21:36:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult: Kill code,
 and hopefully let translation work better.  Not fully tested, 
Message-ID: <20150823193639.1DFD21C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult
Changeset: r79172:ef93711c3900
Date: 2015-08-23 20:25 +0100
http://bitbucket.org/pypy/pypy/changeset/ef93711c3900/

Log:	Kill code, and hopefully let translation work better. Not fully
	tested, because a number of the tests modified were not passing
	before...

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -101,9 +101,6 @@
     def is_object(self):
         return self._is_object
 
-    def as_vtable_size_descr(self):
-        return self
-
     def count_fields_if_immutable(self):
         return heaptracker.count_fields_if_immutable(self.S)
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -59,9 +59,6 @@
         return False
 
 class SizeDescrWithVTable(SizeDescr):
-    def as_vtable_size_descr(self):
-        return self
-
     def is_object(self):
         return True
 
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -8,7 +8,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, lltype, llmemory
 from rpython.rtyper import rclass
@@ -52,7 +51,6 @@
         O = lltype.GcStruct('O', ('parent', rclass.OBJECT),
                                  ('x', lltype.Signed))
         o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -7,7 +7,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -64,7 +63,6 @@
                                  ('x', lltype.Signed))
         o_descr = self.cpu.sizeof(O, True)
         o_vtable = globals()['o_vtable']
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1950,8 +1950,7 @@
         cpu = self.cpu
         t_box, T_box, descr = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
-        heaptracker.register_known_gctype(cpu, vtable, self.T)
+            llmemory.cast_int_to_adr(T_box.getint()), rclass.CLASSTYPE)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         assert r1 != r2
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -123,8 +123,6 @@
         vtable.name = rclass.alloc_array_name(name)
         self.structure_types_and_vtables.append((S, vtable))
         #
-        heaptracker.register_known_gctype(self.cpu, vtable, S)
-        #
         return S, vtable
 
     def get_random_structure(self, r, has_vtable=False):
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -74,7 +74,6 @@
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
         self.assembler.finished(self.callcontrol.callinfocollection)
-        heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -88,57 +88,6 @@
 
 # ____________________________________________________________
 
-VTABLETYPE = rclass.CLASSTYPE
-
-def register_known_gctype(cpu, vtable, STRUCT):
-    # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
-    sizedescr = cpu.sizeof(STRUCT, vtable)
-    assert sizedescr.as_vtable_size_descr() is sizedescr
-    if getattr(sizedescr, '_corresponding_vtable', None):
-        assert sizedescr._corresponding_vtable == vtable
-        return
-    assert lltype.typeOf(vtable) == VTABLETYPE
-    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-        cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._vtable_to_descr_dict = None
-    cpu.tracker._all_size_descrs_with_vtable.append(sizedescr)
-    sizedescr._corresponding_vtable = vtable
-
-def finish_registering(cpu):
-    # annotation hack for small examples which have no vtable at all
-    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
-    pass
-
-def vtable2descr(cpu, vtable):
-    assert lltype.typeOf(vtable) is lltype.Signed
-    vtable = int2adr(vtable)
-    if we_are_translated():
-        # Build the dict {vtable: sizedescr} at runtime.
-        # This is necessary because the 'vtables' are just pointers to
-        # static data, so they can't be used as keys in prebuilt dicts.
-        d = cpu.tracker._vtable_to_descr_dict
-        if d is None:
-            d = cpu.tracker._vtable_to_descr_dict = {}
-            for descr in cpu.tracker._all_size_descrs_with_vtable:
-                key = descr._corresponding_vtable
-                key = llmemory.cast_ptr_to_adr(key)
-                d[key] = descr
-        return d[vtable]
-    else:
-        vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE)
-        for descr in cpu.tracker._all_size_descrs_with_vtable:
-            if descr._corresponding_vtable == vtable:
-                return descr
-        raise KeyError(vtable)
-
-def descr2vtable(cpu, descr):
-    from rpython.jit.metainterp import history
-    assert isinstance(descr, history.AbstractDescr)
-    vtable = descr.as_vtable_size_descr()._corresponding_vtable
-    vtable = llmemory.cast_ptr_to_adr(vtable)
-    return adr2int(vtable)
 
 def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None,
                     get_field_descr=None):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -932,7 +932,6 @@
                     return self._do_builtin_call(op, 'alloc_with_del', [],
                                                  extra=(RESULT, vtable),
                                                  extrakey=STRUCT)
-            heaptracker.register_known_gctype(self.cpu, vtable, STRUCT)
             opname = 'new_with_vtable'
         else:
             opname = 'new'
@@ -1936,10 +1935,6 @@
     # VirtualRefs.
 
     def _handle_virtual_ref_call(self, op, oopspec_name, args):
-        vrefinfo = self.callcontrol.virtualref_info
-        heaptracker.register_known_gctype(self.cpu,
-                                          vrefinfo.jit_virtual_ref_vtable,
-                                          vrefinfo.JIT_VIRTUAL_REF)
         return SpaceOperation(oopspec_name, list(args), op.result)
 
     # -----------
diff --git a/rpython/jit/codewriter/test/test_codewriter.py b/rpython/jit/codewriter/test/test_codewriter.py
--- a/rpython/jit/codewriter/test/test_codewriter.py
+++ b/rpython/jit/codewriter/test/test_codewriter.py
@@ -25,8 +25,6 @@
 class FakeSizeDescr(AbstractDescr):
     def __init__(self, STRUCT):
         self.STRUCT = STRUCT
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeArrayDescr(AbstractDescr):
     def __init__(self, ARRAY):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -39,8 +39,6 @@
         self.oopspecindex = oopspecindex
     def __repr__(self):
         return ''
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeDict(object):
     def __getitem__(self, key):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -50,8 +50,7 @@
         return FakeDescr(('sizedescr', STRUCT))
 
 class FakeDescr(tuple):
-    def as_vtable_size_descr(self):
-        return self
+    pass
 
 class FakeLink:
     args = []
@@ -569,9 +568,6 @@
     op1 = Transformer(cpu).rewrite_operation(op)
     assert op1.opname == 'new_with_vtable'
     assert op1.args == [('sizedescr', S)]
-    #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check]
-    vtable_int = heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtable))
-    assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0]
 
 def test_malloc_new_with_destructor():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -12,7 +12,6 @@
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.logger import LogOperations
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import OpParser
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
@@ -310,14 +309,6 @@
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
     vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
 
-    register_known_gctype(cpu, node_vtable,  NODE)
-    register_known_gctype(cpu, node_vtable2, NODE2)
-    register_known_gctype(cpu, u_vtable,     U)
-    register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
-    register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
-    register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
-    register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
-
     namespace = locals()
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,5 +1,4 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.codewriter.heaptracker import descr2vtable
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
@@ -202,7 +201,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr()))
+        known_class = ConstInt(op.getdescr().get_vtable())
         self.make_virtual(known_class, op, op.getdescr())
 
     def optimize_NEW(self, op):
diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -597,8 +597,7 @@
                 self._no = no
 
         class FakeDescr:
-            def as_vtable_size_descr(self):
-                return self
+            pass
 
         class FakeCPU(object):
             supports_floats = False

From noreply at buildbot.pypy.org  Sun Aug 23 21:40:36 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 23 Aug 2015 21:40:36 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: hg merge optresult
Message-ID: <20150823194036.285251C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79173:d28b92c9f186
Date: 2015-08-23 21:40 +0200
http://bitbucket.org/pypy/pypy/changeset/d28b92c9f186/

Log:	hg merge optresult

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -101,9 +101,6 @@
     def is_object(self):
         return self._is_object
 
-    def as_vtable_size_descr(self):
-        return self
-
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
             self._corresponding_vtable))
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -59,9 +59,6 @@
         return False
 
 class SizeDescrWithVTable(SizeDescr):
-    def as_vtable_size_descr(self):
-        return self
-
     def is_object(self):
         return True
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -121,7 +121,8 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- GETFIELD_GC ----------
-            if op.getopnum() == rop.GETFIELD_GC:
+            if op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F,
+                                 rop.GETFIELD_GC_R):
                 self.handle_getfield_gc(op)
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
@@ -194,7 +195,7 @@
         followed by a bunch of 'setfields', and the 'pending_zeros'
         optimization we do here is meant for this case."""
         self.emit_pending_zeros()
-        self.newops.append(op)
+        self.emit_op(op)
 
     # ----------
 
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -8,7 +8,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, lltype, llmemory
 from rpython.rtyper import rclass
@@ -52,7 +51,6 @@
         O = lltype.GcStruct('O', ('parent', rclass.OBJECT),
                                  ('x', lltype.Signed))
         o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -7,7 +7,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -64,7 +63,6 @@
                                  ('x', lltype.Signed))
         o_descr = self.cpu.sizeof(O, True)
         o_vtable = globals()['o_vtable']
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1950,8 +1950,7 @@
         cpu = self.cpu
         t_box, T_box, descr = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
-        heaptracker.register_known_gctype(cpu, vtable, self.T)
+            llmemory.cast_int_to_adr(T_box.getint()), rclass.CLASSTYPE)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         assert r1 != r2
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -123,8 +123,6 @@
         vtable.name = rclass.alloc_array_name(name)
         self.structure_types_and_vtables.append((S, vtable))
         #
-        heaptracker.register_known_gctype(self.cpu, vtable, S)
-        #
         return S, vtable
 
     def get_random_structure(self, r, has_vtable=False):
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -74,7 +74,6 @@
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
         self.assembler.finished(self.callcontrol.callinfocollection)
-        heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -88,57 +88,6 @@
 
 # ____________________________________________________________
 
-VTABLETYPE = rclass.CLASSTYPE
-
-def register_known_gctype(cpu, vtable, STRUCT):
-    # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
-    sizedescr = cpu.sizeof(STRUCT, vtable)
-    assert sizedescr.as_vtable_size_descr() is sizedescr
-    if getattr(sizedescr, '_corresponding_vtable', None):
-        assert sizedescr._corresponding_vtable == vtable
-        return
-    assert lltype.typeOf(vtable) == VTABLETYPE
-    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-        cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._vtable_to_descr_dict = None
-    cpu.tracker._all_size_descrs_with_vtable.append(sizedescr)
-    sizedescr._corresponding_vtable = vtable
-
-def finish_registering(cpu):
-    # annotation hack for small examples which have no vtable at all
-    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
-    pass
-
-def vtable2descr(cpu, vtable):
-    assert lltype.typeOf(vtable) is lltype.Signed
-    vtable = int2adr(vtable)
-    if we_are_translated():
-        # Build the dict {vtable: sizedescr} at runtime.
-        # This is necessary because the 'vtables' are just pointers to
-        # static data, so they can't be used as keys in prebuilt dicts.
-        d = cpu.tracker._vtable_to_descr_dict
-        if d is None:
-            d = cpu.tracker._vtable_to_descr_dict = {}
-            for descr in cpu.tracker._all_size_descrs_with_vtable:
-                key = descr._corresponding_vtable
-                key = llmemory.cast_ptr_to_adr(key)
-                d[key] = descr
-        return d[vtable]
-    else:
-        vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE)
-        for descr in cpu.tracker._all_size_descrs_with_vtable:
-            if descr._corresponding_vtable == vtable:
-                return descr
-        raise KeyError(vtable)
-
-def descr2vtable(cpu, descr):
-    from rpython.jit.metainterp import history
-    assert isinstance(descr, history.AbstractDescr)
-    vtable = descr.as_vtable_size_descr()._corresponding_vtable
-    vtable = llmemory.cast_ptr_to_adr(vtable)
-    return adr2int(vtable)
 
 def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None,
                     get_field_descr=None):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -932,7 +932,6 @@
                     return self._do_builtin_call(op, 'alloc_with_del', [],
                                                  extra=(RESULT, vtable),
                                                  extrakey=STRUCT)
-            heaptracker.register_known_gctype(self.cpu, vtable, STRUCT)
             opname = 'new_with_vtable'
         else:
             opname = 'new'
@@ -1936,10 +1935,6 @@
     # VirtualRefs.
 
     def _handle_virtual_ref_call(self, op, oopspec_name, args):
-        vrefinfo = self.callcontrol.virtualref_info
-        heaptracker.register_known_gctype(self.cpu,
-                                          vrefinfo.jit_virtual_ref_vtable,
-                                          vrefinfo.JIT_VIRTUAL_REF)
         return SpaceOperation(oopspec_name, list(args), op.result)
 
     # -----------
diff --git a/rpython/jit/codewriter/test/test_codewriter.py b/rpython/jit/codewriter/test/test_codewriter.py
--- a/rpython/jit/codewriter/test/test_codewriter.py
+++ b/rpython/jit/codewriter/test/test_codewriter.py
@@ -25,8 +25,6 @@
 class FakeSizeDescr(AbstractDescr):
     def __init__(self, STRUCT):
         self.STRUCT = STRUCT
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeArrayDescr(AbstractDescr):
     def __init__(self, ARRAY):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -39,8 +39,6 @@
         self.oopspecindex = oopspecindex
     def __repr__(self):
         return ''
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeDict(object):
     def __getitem__(self, key):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -50,8 +50,7 @@
         return FakeDescr(('sizedescr', STRUCT))
 
 class FakeDescr(tuple):
-    def as_vtable_size_descr(self):
-        return self
+    pass
 
 class FakeLink:
     args = []
@@ -569,9 +568,6 @@
     op1 = Transformer(cpu).rewrite_operation(op)
     assert op1.opname == 'new_with_vtable'
     assert op1.args == [('sizedescr', S)]
-    #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check]
-    vtable_int = heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtable))
-    assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0]
 
 def test_malloc_new_with_destructor():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -242,6 +242,11 @@
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
 
+    def _get_buffer(self):
+        buffer = self.buffer
+        assert buffer is not None
+        return buffer
+
     def getitem_raw(self, offset, itemsize, descr):
         if not self.is_virtual():
             raise InvalidRawOperation
@@ -249,37 +254,39 @@
             # for the test above: it's not enough to check is_virtual()
             # on the original object, because it might be a VRawSliceValue
             # instead.  If it is a virtual one, then we'll reach here anway.
-        return self.buffer.read_value(offset, itemsize, descr)
+        return self._get_buffer().read_value(offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         if not self.is_virtual():
             raise InvalidRawOperation
-        self.buffer.write_value(offset, itemsize, descr, itemop)
+        self._get_buffer().write_value(offset, itemsize, descr, itemop)
 
     def is_virtual(self):
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
         self.size = -1
-        for i in range(len(self.buffer.offsets)):
+        buffer = self._get_buffer()
+        for i in range(len(buffer.offsets)):
             # write the value
-            offset = self.buffer.offsets[i]
-            descr = self.buffer.descrs[i]
-            itembox = self.buffer.values[i]
+            offset = buffer.offsets[i]
+            descr = buffer.descrs[i]
+            itembox = buffer.values[i]
             setfield_op = ResOperation(rop.RAW_STORE,
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(setfield_op)
 
     def _visitor_walk_recursive(self, op, visitor, optimizer):
-        itemboxes = self.buffer.values
+        itemboxes = self._get_buffer().values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
+        buffer = self._get_buffer()
         return visitor.visit_vrawbuffer(self.size,
-                                        self.buffer.offsets[:],
-                                        self.buffer.descrs[:])
+                                        buffer.offsets[:],
+                                        buffer.descrs[:])
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -12,7 +12,6 @@
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.logger import LogOperations
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import OpParser
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
@@ -352,17 +351,6 @@
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
     vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
 
-    register_known_gctype(cpu, node_vtable,  NODE)
-    register_known_gctype(cpu, node_vtable2, NODE2)
-    register_known_gctype(cpu, node_vtable3, NODE3)
-    register_known_gctype(cpu, u_vtable,     U)
-    register_known_gctype(cpu, simple_vtable,     SIMPLE)
-    register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
-    register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
-    register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
-    register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
-    register_known_gctype(cpu, w_root_vtable, W_ROOT)
-
     namespace = locals()
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,5 +1,4 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.codewriter.heaptracker import descr2vtable
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
@@ -199,7 +198,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr()))
+        known_class = ConstInt(op.getdescr().get_vtable())
         self.make_virtual(known_class, op, op.getdescr())
 
     def optimize_NEW(self, op):
diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -597,8 +597,7 @@
                 self._no = no
 
         class FakeDescr:
-            def as_vtable_size_descr(self):
-                return self
+            pass
 
         class FakeCPU(object):
             supports_floats = False

From noreply at buildbot.pypy.org  Sun Aug 23 21:44:37 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Sun, 23 Aug 2015 21:44:37 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: need the 4th element in the tuple,
 the reference to descr.version is removed!
Message-ID: <20150823194437.80D0F1C0FF7@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79174:c0ed1ea7fc3b
Date: 2015-08-23 21:44 +0200
http://bitbucket.org/pypy/pypy/changeset/c0ed1ea7fc3b/

Log:	need the 4th element in the tuple, the reference to descr.version is
	removed!

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -598,8 +598,7 @@
             Why? Because this only generates a very small junk of memory, instead of
             duplicating the loop assembler for each faildescr!
         """
-        asminfo, bridge_faildescr, looptoken = version._compiled
-        compiled_version = bridge_faildescr.version
+        asminfo, bridge_faildescr, compiled_version, looptoken = version._compiled
         assert isinstance(bridge_faildescr, ResumeGuardDescr)
         assert isinstance(faildescr, ResumeGuardDescr)
         assert asminfo.rawstart != 0
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -209,7 +209,7 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version._compiled = (asminfo, faildescr, jitcell_token)
+            version._compiled = (asminfo, faildescr, faildescr.version, jitcell_token)
             faildescr.version = None
         # stitch to the trace loop
         for lv in loop.versions:
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -746,7 +746,7 @@
     """
     def __init__(self, loop):
         self.faildescrs = []
-        self._compiled = (None,None,None)
+        self._compiled = (None,None,None,None)
         if loop:
             self.operations = self.copy_operations(loop.operations) 
             idx = index_of_first(rop.LABEL, self.operations)

From noreply at buildbot.pypy.org  Sun Aug 23 22:07:57 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 23 Aug 2015 22:07:57 +0200 (CEST)
Subject: [pypy-commit] pypy default: add freebsd builder to release builds,
 allow setting user name in force_builds
Message-ID: <20150823200757.7D8861C0FF7@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79175:fd574a88c241
Date: 2015-08-23 23:08 +0300
http://bitbucket.org/pypy/pypy/changeset/fd574a88c241/

Log:	add freebsd builder to release builds, allow setting user name in
	force_builds

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -31,15 +31,14 @@
   and add the new file to  pypy/doc/index-of-whatsnew.rst
 * go to pypy/tool/release and run
   ``force-builds.py ``
-  The following binaries should be built, however, we need more buildbots
- - JIT: windows, linux, os/x, armhf, armel
- - no JIT: windows, linux, os/x
- - sandbox: linux, os/x
+  The following JIT binaries should be built, however, we need more buildbots
+  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+  freebsd64 
 
 * wait for builds to complete, make sure there are no failures
 * download the builds, repackage binaries. Tag the release version
   and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tools to do this. 
+  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
 
   Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
   and to cobra, as some packagers prefer a clearly labeled source package
diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py
--- a/pypy/tool/release/force-builds.py
+++ b/pypy/tool/release/force-builds.py
@@ -28,6 +28,7 @@
 #    'pypy-c-app-level-win-x86-32',
     'pypy-c-jit-linux-x86-32',
     'pypy-c-jit-linux-x86-64',
+    'pypy-c-jit-freebsd-9-x86-64',
     'pypy-c-jit-macosx-x86-64',
     'pypy-c-jit-win-x86-32',
     'build-pypy-c-jit-linux-armhf-raring',
@@ -42,7 +43,7 @@
         import pwd
         return pwd.getpwuid(os.getuid())[0]
 
-def main(branch, server):
+def main(branch, server, user):
     #XXX: handle release tags
     #XXX: handle validity checks
     lock = defer.DeferredLock()
@@ -56,7 +57,7 @@
         print 'Forcing', builder, '...'
         url = "http://" + server + "/builders/" + builder + "/force"
         args = [
-            ('username', get_user()),
+            ('username', user),
             ('revision', ''),
             ('forcescheduler', 'Force Scheduler'),
             ('submit', 'Force Build'),
@@ -78,7 +79,8 @@
     parser = optparse.OptionParser()
     parser.add_option("-b", "--branch", help="branch to build", default='')
     parser.add_option("-s", "--server", help="buildbot server", default="buildbot.pypy.org")
+    parser.add_option("-u", "--user", help="user name to report", default=get_user())
     (options, args) = parser.parse_args()
     if  not options.branch:
         parser.error("branch option required")
-    main(options.branch, options.server)
+    main(options.branch, options.server, user=options.user)
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,12 +1,12 @@
 # Edit these appropriately before running this script
 maj=2
 min=6
-rev=0
+rev=1
 # This script will download latest builds from the buildmaster, rename the top
 # level directory, and repackage ready to be uploaded to bitbucket. It will also
 # download source, assuming a tag for the release already exists, and repackage them.
 
-for plat in linux linux64 linux-armhf-raspbian linux-armhf-raring linux-armel osx64
+for plat in linux linux64 linux-armhf-raspbian linux-armhf-raring linux-armel osx64 freebsd64
   do
     wget http://buildbot.pypy.org/nightly/release-$maj.$min.x/pypy-c-jit-latest-$plat.tar.bz2
     tar -xf pypy-c-jit-latest-$plat.tar.bz2

From noreply at buildbot.pypy.org  Mon Aug 24 10:12:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 10:12:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: rework the retracing
Message-ID: <20150824081245.ABF5B1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79176:0e3a0ea12d66
Date: 2015-08-24 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/0e3a0ea12d66/

Log:	rework the retracing

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -289,18 +289,11 @@
                               original_jitcell_token=jitcell_token)
     start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs,
                                descr=start_descr)
-    mid_descr_token = TargetToken(jitcell_token,
-                                  original_jitcell_token=jitcell_token)
-    mid_descr_token.short_preamble = loop_info.short_preamble
-    mid_descr_token.virtual_state = start_state.virtual_state
-    mid_label = ResOperation(rop.LABEL, loop_info.label_args,
-                             descr=mid_descr_token)
     # XXX assign short preamble and virtual state
-    loop_ops[-1].setdescr(mid_descr_token)
     loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
-                       [mid_label] + loop_ops)
+                       [loop_info.label_op] + loop_ops)
     loop.check_consistency()
-    jitcell_token.target_tokens = [start_descr, mid_descr_token]
+    jitcell_token.target_tokens = [start_descr] + jitcell_token.target_tokens
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
     return start_descr
@@ -333,33 +326,13 @@
         loop_info, loop_ops = optimize_trace(metainterp_sd, jitdriver_sd,
                                              loop_data)
     except InvalidLoop:
+        # Fall back on jumping to preamble
         xxx
-        # Fall back on jumping to preamble
-        target_token = label.getdescr()
-        assert isinstance(target_token, TargetToken)
-        part.operations = [orignial_label] + \
-                          [ResOperation(rop.JUMP, inputargs[:],
-                                        descr=loop_jitcell_token)]
-        try:
-            optimize_trace(metainterp_sd, jitdriver_sd, part,
-                           jitdriver_sd.warmstate.enable_opts,
-                           inline_short_preamble=False, start_state=start_state,
-                           export_state=False)
-        except InvalidLoop:
-            xxx # XXX forget optimizations
-            return None
 
     loop = partial_trace
-    target_token = TargetToken(loop_jitcell_token)
-    target_token.original_jitcell_token = loop_jitcell_token
-    target_token.short_preamble = loop_info.short_preamble
-    target_token.virtual_state = start_state.virtual_state
-    loop_ops[-1].setdescr(target_token)
-    mid_label = ResOperation(rop.LABEL, loop_info.label_args,
-                             descr=target_token)
-    loop.operations = (loop.operations + loop_info.extra_same_as + [mid_label]
+    loop.operations = (loop.operations + loop_info.extra_same_as +
+                       [loop_info.label_op]
                        + loop_ops)
-    loop_jitcell_token.target_tokens.append(target_token)
 
     #quasi_immutable_deps = {}
     #if loop.quasi_immutable_deps:
@@ -369,6 +342,7 @@
     #if quasi_immutable_deps:
     #    loop.quasi_immutable_deps = quasi_immutable_deps
 
+    target_token = loop.operations[-1].getdescr()
     resumekey.compile_and_attach(metainterp, loop)
 
     record_loop_or_bridge(metainterp_sd, loop)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -336,6 +336,8 @@
                 sb = self.optunroll.short_preamble_producer
                 sb.add_preamble_op(preamble_op)
         if info is not None:
+            if op.type == 'i' and info.is_constant():
+                return ConstInt(info.getint())
             return info.force_box(op, self)
         return op
 
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -393,9 +393,9 @@
         self.used_boxes.append(preamble_op.op)            
         self.short_preamble_jump.append(preamble_op.preamble_op)
 
-    def build_short_preamble(self):
+    def build_short_preamble(self, sb_jump):
         label_op = ResOperation(rop.LABEL, self.short_inputargs[:])
-        jump_op = ResOperation(rop.JUMP, self.short_preamble_jump)
+        jump_op = ResOperation(rop.JUMP, sb_jump)
         TreeLoop.check_consistency_of(self.short_inputargs,
                                       self.short + [jump_op], check_descr=False)
         return [label_op] + self.short + [jump_op]
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8122,7 +8122,7 @@
         i4 = int_add(i3, i3)
         i5 = int_add(i4, i4)
         i6 = int_add(i5, i5)
-        jump(i6, i3)
+        jump(i6)
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -483,7 +483,9 @@
         jump_op = loop.operations[-1]
         assert jump_op.getopnum() == rop.JUMP
         ops = loop.operations[:-1]
-        start_label = ResOperation(rop.LABEL, loop.inputargs)
+        jump_op.setdescr(JitCellToken())
+        start_label = ResOperation(rop.LABEL, loop.inputargs,
+                                   jump_op.getdescr())
         end_label = jump_op.copy_and_change(opnum=rop.LABEL)
         call_pure_results = self._convert_call_pure_results(call_pure_results)
         preamble_data = compile.LoopCompileData(start_label, end_label, ops,
@@ -497,12 +499,11 @@
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
         start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs)
-        emit_end_label = ResOperation(rop.LABEL, loop_info.label_args)
         preamble.operations = ([start_label] + preamble_ops +
-                               loop_info.extra_same_as + [emit_end_label])
-        loop.inputargs = loop_info.label_args[:]
-        loop.operations = [emit_end_label] + ops
-        return Info(preamble, loop_info.short_preamble,
+                               loop_info.extra_same_as + [loop_info.label_op])
+        loop.inputargs = loop_info.label_op.getarglist()[:]
+        loop.operations = [loop_info.label_op] + ops
+        return Info(preamble, loop_info.target_token.short_preamble,
                     start_state.virtual_state)
 
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,6 +1,6 @@
 
 import sys
-from rpython.jit.metainterp.history import Const
+from rpython.jit.metainterp.history import Const, TargetToken
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
      ShortPreambleBuilder, PreambleOp
 from rpython.jit.metainterp.optimizeopt import info, intutils
@@ -58,10 +58,10 @@
         elif isinstance(preamble_info, intutils.IntBound):
             if preamble_info.lower > MININT/2 or preamble_info.upper < MAXINT/2:
                 intbound = self.getintbound(op)
-                if preamble_info.lower > MININT/2:
+                if preamble_info.has_lower and preamble_info.lower > MININT/2:
                     intbound.has_lower = True
                     intbound.lower = preamble_info.lower
-                if preamble_info.upper < MAXINT/2:
+                if preamble_info.has_upper and preamble_info.upper < MAXINT/2:
                     intbound.has_upper = True
                     intbound.upper = preamble_info.upper
 
@@ -108,29 +108,24 @@
         label_args = state.virtual_state.make_inputargs(
             start_label.getarglist(), self.optimizer)
         self.optimizer.init_inparg_dict_from(label_args)
-        self.optimizer.propagate_all_forward(start_label.getarglist()[:], ops,
-                                             call_pure_results, False)
-        orig_jump_args = [self.get_box_replacement(op)
-                     for op in end_jump.getarglist()]
-        jump_args = state.virtual_state.make_inputargs(orig_jump_args,
-                                    self.optimizer, force_boxes=True)
-        pass_to_short = state.virtual_state.make_inputargs(orig_jump_args,
-                                    self.optimizer, force_boxes=True,
-                                    append_virtuals=True)
-        sb = self.short_preamble_producer
-        self.optimizer._clean_optimization_info(sb.short_inputargs)
-        extra_jump_args = self.inline_short_preamble(pass_to_short,
-                                sb.short_inputargs, sb.short,
-                                sb.short_preamble_jump,
-                                self.optimizer.patchguardop)
-        # remove duplicates, removes stuff from used boxes too
-        label_args, jump_args = self.filter_extra_jump_args(
-            label_args + self.short_preamble_producer.used_boxes,
-            jump_args + extra_jump_args)
-        jump_op = ResOperation(rop.JUMP, jump_args)
-        self.optimizer.send_extra_operation(jump_op)
-        return (UnrollInfo(self.short_preamble_producer.build_short_preamble(),
-                           label_args,
+        info, _ = self.optimizer.propagate_all_forward(
+            start_label.getarglist()[:], ops, call_pure_results, False)
+        label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
+        target_token, extra = self.finalize_short_preamble(label_op,
+                                                    state.virtual_state)
+        label_op.setdescr(target_token)
+        label_op.initarglist(label_op.getarglist() + extra)
+        # force the boxes for virtual state to match
+        x = state.virtual_state.make_inputargs(
+            [self.get_box_replacement(x) for x in end_jump.getarglist()],
+            self.optimizer, force_boxes=True)
+        new_virtual_state = self.jump_to_existing_trace(end_jump)
+        if new_virtual_state is not None:
+            res = self.jump_to_preamble(start_label.getdescr(), end_jump,
+                                         info)
+            xxx
+            #return new_virtual_state, self.optimizer._newoperations
+        return (UnrollInfo(target_token, label_op,
                            self.short_preamble_producer.extra_same_as),
                 self.optimizer._newoperations)
 
@@ -145,7 +140,7 @@
         cell_token = jump_op.getdescr()
         if not inline_short_preamble or len(cell_token.target_tokens) == 1:
             return self.jump_to_preamble(cell_token, jump_op, info)
-        vs = self.jump_to_existing_trace(jump_op, inline_short_preamble)
+        vs = self.jump_to_existing_trace(jump_op)
         if vs is None:
             return info, self.optimizer._newoperations[:]
         warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc
@@ -171,6 +166,32 @@
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
+    def finalize_short_preamble(self, label_op, virtual_state):
+        sb = self.short_preamble_producer
+        self.optimizer._clean_optimization_info(sb.short_inputargs)
+        d = {}
+        for arg in label_op.getarglist():
+            d[arg] = None
+        new_used_boxes = []
+        new_sb_jump = []
+        for i in range(len(sb.used_boxes)):
+            ub = sb.used_boxes[i]
+            if ub in d:
+                continue
+            new_used_boxes.append(ub)
+            new_sb_jump.append(sb.short_preamble_jump[i])
+        short_preamble = sb.build_short_preamble(new_sb_jump)
+        jitcelltoken = label_op.getdescr()
+        if jitcelltoken.target_tokens is None:
+            jitcelltoken.target_tokens = []
+        target_token = TargetToken(jitcelltoken,
+                                   original_jitcell_token=jitcelltoken)
+        target_token.original_jitcell_token = jitcelltoken
+        target_token.virtual_state = virtual_state
+        target_token.short_preamble = short_preamble
+        jitcelltoken.target_tokens.append(target_token)
+        return target_token, new_used_boxes
+
     def jump_to_preamble(self, cell_token, jump_op, info):
         assert cell_token.target_tokens[0].virtual_state is None
         jump_op = jump_op.copy_and_change(rop.JUMP,
@@ -179,7 +200,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op, inline_short_preamble):
+    def jump_to_existing_trace(self, jump_op):
         jitcelltoken = jump_op.getdescr()
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
         virtual_state = self.get_virtual_state(args)
@@ -203,7 +224,7 @@
                 continue
             short_preamble = target_token.short_preamble
             pass_to_short = target_virtual_state.make_inputargs(args,
-                self.optimizer, force_boxes=True, append_virtuals=True)
+                self.optimizer, append_virtuals=True)
             args = target_virtual_state.make_inputargs(args,
                 self.optimizer)
             extra = self.inline_short_preamble(pass_to_short,
@@ -215,22 +236,6 @@
             return None # explicit because the return can be non-None
         return virtual_state
 
-    def filter_extra_jump_args(self, label_args, jump_args):
-        label_args = [self.get_box_replacement(x, True) for x in label_args]
-        jump_args = [self.get_box_replacement(x) for x in jump_args]
-        new_label_args = []
-        new_jump_args = []
-        assert len(label_args) == len(jump_args)
-        d = {}
-        for i in range(len(label_args)):
-            arg = label_args[i]
-            if arg in d:
-                continue
-            new_label_args.append(arg)
-            new_jump_args.append(jump_args[i])
-            d[arg] = None
-        return new_label_args, new_jump_args
-
     def inline_short_preamble(self, jump_args, short_inputargs, short_ops,
                               short_jump_op, patchguardop):
         try:
@@ -318,13 +323,13 @@
 class UnrollInfo(LoopInfo):
     """ A state after optimizing the peeled loop, contains the following:
 
-    * short_preamble - list of operations that go into short preamble
-    * label_args - additional things to put in the label
+    * target_token - generated target token
+    * label_args - label operations at the beginning
     * extra_same_as - list of extra same as to add at the end of the preamble
     """
-    def __init__(self, short_preamble, label_args, extra_same_as):
-        self.short_preamble = short_preamble
-        self.label_args = label_args
+    def __init__(self, target_token, label_op, extra_same_as):
+        self.target_token = target_token
+        self.label_op = label_op
         self.extra_same_as = extra_same_as
 
     def final(self):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -133,12 +133,18 @@
             if box is not None:
                 fieldbox = opinfo._fields[self.fielddescrs[i].get_index()]
                 # must be there
-                fieldinfo = get_forwarded(fieldbox)
+                if fieldbox is not None:
+                    fieldinfo = get_forwarded(fieldbox)
+                else:
+                    fieldinfo = None
             else:
                 fieldbox = None
                 fieldinfo = None
-            self.fieldstate[i].generate_guards(other.fieldstate[i], fieldbox,
-                                               fieldinfo, state)
+            # XXX all those ifs are for tests, not sure what to do
+            if self.fieldstate[i] is not None:
+                self.fieldstate[i].generate_guards(other.fieldstate[i],
+                                                   fieldbox,
+                                                   fieldinfo, state)
 
 
     def _generalization_of_structpart(self, other):

From noreply at buildbot.pypy.org  Mon Aug 24 10:12:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 10:12:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: merge
Message-ID: <20150824081248.398851C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79177:cc2b292ce9b0
Date: 2015-08-24 10:12 +0200
http://bitbucket.org/pypy/pypy/changeset/cc2b292ce9b0/

Log:	merge

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -101,9 +101,6 @@
     def is_object(self):
         return self._is_object
 
-    def as_vtable_size_descr(self):
-        return self
-
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
             self._corresponding_vtable))
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -59,9 +59,6 @@
         return False
 
 class SizeDescrWithVTable(SizeDescr):
-    def as_vtable_size_descr(self):
-        return self
-
     def is_object(self):
         return True
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -121,7 +121,8 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- GETFIELD_GC ----------
-            if op.getopnum() == rop.GETFIELD_GC:
+            if op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F,
+                                 rop.GETFIELD_GC_R):
                 self.handle_getfield_gc(op)
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
@@ -194,7 +195,7 @@
         followed by a bunch of 'setfields', and the 'pending_zeros'
         optimization we do here is meant for this case."""
         self.emit_pending_zeros()
-        self.newops.append(op)
+        self.emit_op(op)
 
     # ----------
 
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -8,7 +8,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, lltype, llmemory
 from rpython.rtyper import rclass
@@ -52,7 +51,6 @@
         O = lltype.GcStruct('O', ('parent', rclass.OBJECT),
                                  ('x', lltype.Signed))
         o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -7,7 +7,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -64,7 +63,6 @@
                                  ('x', lltype.Signed))
         o_descr = self.cpu.sizeof(O, True)
         o_vtable = globals()['o_vtable']
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1950,8 +1950,7 @@
         cpu = self.cpu
         t_box, T_box, descr = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
-        heaptracker.register_known_gctype(cpu, vtable, self.T)
+            llmemory.cast_int_to_adr(T_box.getint()), rclass.CLASSTYPE)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         assert r1 != r2
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -123,8 +123,6 @@
         vtable.name = rclass.alloc_array_name(name)
         self.structure_types_and_vtables.append((S, vtable))
         #
-        heaptracker.register_known_gctype(self.cpu, vtable, S)
-        #
         return S, vtable
 
     def get_random_structure(self, r, has_vtable=False):
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -74,7 +74,6 @@
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
         self.assembler.finished(self.callcontrol.callinfocollection)
-        heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -88,57 +88,6 @@
 
 # ____________________________________________________________
 
-VTABLETYPE = rclass.CLASSTYPE
-
-def register_known_gctype(cpu, vtable, STRUCT):
-    # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
-    sizedescr = cpu.sizeof(STRUCT, vtable)
-    assert sizedescr.as_vtable_size_descr() is sizedescr
-    if getattr(sizedescr, '_corresponding_vtable', None):
-        assert sizedescr._corresponding_vtable == vtable
-        return
-    assert lltype.typeOf(vtable) == VTABLETYPE
-    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-        cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._vtable_to_descr_dict = None
-    cpu.tracker._all_size_descrs_with_vtable.append(sizedescr)
-    sizedescr._corresponding_vtable = vtable
-
-def finish_registering(cpu):
-    # annotation hack for small examples which have no vtable at all
-    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
-    pass
-
-def vtable2descr(cpu, vtable):
-    assert lltype.typeOf(vtable) is lltype.Signed
-    vtable = int2adr(vtable)
-    if we_are_translated():
-        # Build the dict {vtable: sizedescr} at runtime.
-        # This is necessary because the 'vtables' are just pointers to
-        # static data, so they can't be used as keys in prebuilt dicts.
-        d = cpu.tracker._vtable_to_descr_dict
-        if d is None:
-            d = cpu.tracker._vtable_to_descr_dict = {}
-            for descr in cpu.tracker._all_size_descrs_with_vtable:
-                key = descr._corresponding_vtable
-                key = llmemory.cast_ptr_to_adr(key)
-                d[key] = descr
-        return d[vtable]
-    else:
-        vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE)
-        for descr in cpu.tracker._all_size_descrs_with_vtable:
-            if descr._corresponding_vtable == vtable:
-                return descr
-        raise KeyError(vtable)
-
-def descr2vtable(cpu, descr):
-    from rpython.jit.metainterp import history
-    assert isinstance(descr, history.AbstractDescr)
-    vtable = descr.as_vtable_size_descr()._corresponding_vtable
-    vtable = llmemory.cast_ptr_to_adr(vtable)
-    return adr2int(vtable)
 
 def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None,
                     get_field_descr=None):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -932,7 +932,6 @@
                     return self._do_builtin_call(op, 'alloc_with_del', [],
                                                  extra=(RESULT, vtable),
                                                  extrakey=STRUCT)
-            heaptracker.register_known_gctype(self.cpu, vtable, STRUCT)
             opname = 'new_with_vtable'
         else:
             opname = 'new'
@@ -1936,10 +1935,6 @@
     # VirtualRefs.
 
     def _handle_virtual_ref_call(self, op, oopspec_name, args):
-        vrefinfo = self.callcontrol.virtualref_info
-        heaptracker.register_known_gctype(self.cpu,
-                                          vrefinfo.jit_virtual_ref_vtable,
-                                          vrefinfo.JIT_VIRTUAL_REF)
         return SpaceOperation(oopspec_name, list(args), op.result)
 
     # -----------
diff --git a/rpython/jit/codewriter/test/test_codewriter.py b/rpython/jit/codewriter/test/test_codewriter.py
--- a/rpython/jit/codewriter/test/test_codewriter.py
+++ b/rpython/jit/codewriter/test/test_codewriter.py
@@ -25,8 +25,6 @@
 class FakeSizeDescr(AbstractDescr):
     def __init__(self, STRUCT):
         self.STRUCT = STRUCT
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeArrayDescr(AbstractDescr):
     def __init__(self, ARRAY):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -39,8 +39,6 @@
         self.oopspecindex = oopspecindex
     def __repr__(self):
         return ''
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeDict(object):
     def __getitem__(self, key):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -50,8 +50,7 @@
         return FakeDescr(('sizedescr', STRUCT))
 
 class FakeDescr(tuple):
-    def as_vtable_size_descr(self):
-        return self
+    pass
 
 class FakeLink:
     args = []
@@ -569,9 +568,6 @@
     op1 = Transformer(cpu).rewrite_operation(op)
     assert op1.opname == 'new_with_vtable'
     assert op1.args == [('sizedescr', S)]
-    #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check]
-    vtable_int = heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtable))
-    assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0]
 
 def test_malloc_new_with_destructor():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -242,6 +242,11 @@
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
 
+    def _get_buffer(self):
+        buffer = self.buffer
+        assert buffer is not None
+        return buffer
+
     def getitem_raw(self, offset, itemsize, descr):
         if not self.is_virtual():
             raise InvalidRawOperation
@@ -249,37 +254,39 @@
             # for the test above: it's not enough to check is_virtual()
             # on the original object, because it might be a VRawSliceValue
             # instead.  If it is a virtual one, then we'll reach here anway.
-        return self.buffer.read_value(offset, itemsize, descr)
+        return self._get_buffer().read_value(offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         if not self.is_virtual():
             raise InvalidRawOperation
-        self.buffer.write_value(offset, itemsize, descr, itemop)
+        self._get_buffer().write_value(offset, itemsize, descr, itemop)
 
     def is_virtual(self):
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
         self.size = -1
-        for i in range(len(self.buffer.offsets)):
+        buffer = self._get_buffer()
+        for i in range(len(buffer.offsets)):
             # write the value
-            offset = self.buffer.offsets[i]
-            descr = self.buffer.descrs[i]
-            itembox = self.buffer.values[i]
+            offset = buffer.offsets[i]
+            descr = buffer.descrs[i]
+            itembox = buffer.values[i]
             setfield_op = ResOperation(rop.RAW_STORE,
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(setfield_op)
 
     def _visitor_walk_recursive(self, op, visitor, optimizer):
-        itemboxes = self.buffer.values
+        itemboxes = self._get_buffer().values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
+        buffer = self._get_buffer()
         return visitor.visit_vrawbuffer(self.size,
-                                        self.buffer.offsets[:],
-                                        self.buffer.descrs[:])
+                                        buffer.offsets[:],
+                                        buffer.descrs[:])
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -12,7 +12,6 @@
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.logger import LogOperations
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import OpParser
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
@@ -352,17 +351,6 @@
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
     vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
 
-    register_known_gctype(cpu, node_vtable,  NODE)
-    register_known_gctype(cpu, node_vtable2, NODE2)
-    register_known_gctype(cpu, node_vtable3, NODE3)
-    register_known_gctype(cpu, u_vtable,     U)
-    register_known_gctype(cpu, simple_vtable,     SIMPLE)
-    register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
-    register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
-    register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
-    register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
-    register_known_gctype(cpu, w_root_vtable, W_ROOT)
-
     namespace = locals()
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,5 +1,4 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.codewriter.heaptracker import descr2vtable
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
@@ -199,7 +198,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr()))
+        known_class = ConstInt(op.getdescr().get_vtable())
         self.make_virtual(known_class, op, op.getdescr())
 
     def optimize_NEW(self, op):
diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -597,8 +597,7 @@
                 self._no = no
 
         class FakeDescr:
-            def as_vtable_size_descr(self):
-                return self
+            pass
 
         class FakeCPU(object):
             supports_floats = False

From noreply at buildbot.pypy.org  Mon Aug 24 10:17:29 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 24 Aug 2015 10:17:29 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: hack to re-add blackhole interp
	caching
Message-ID: <20150824081729.79CBD1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8-gcc
Changeset: r79178:b9bb7b46ca8e
Date: 2015-08-24 10:20 +0200
http://bitbucket.org/pypy/pypy/changeset/b9bb7b46ca8e/

Log:	hack to re-add blackhole interp caching

	Currently leaks the interps of dead threads (needs fix). It improves
	the performance of 'pixie' annotation > rpython --thread --no-
	shared --gcrootfinder=shadowstack --continuation -O2 --annotate
	target.py from ~63s down to ~42s (pypy2.6 ~29s).

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -39,6 +39,40 @@
 
 # ____________________________________________________________
 
+from rpython.rlib import rthread#, rstm
+# from rpython.rtyper.annlowlevel import cast_gcref_to_instance
+# from rpython.rtyper.annlowlevel import cast_instance_to_gcref
+
+class BlackholeInterpreterCache(object):
+    # XXX: How to free bhis from dead threads?
+    #      A callback from rthread.gc_thread_die?
+    #      stm_hashtable to avoid conflicts when starting many threads?
+
+    def __init__(self):
+        assert rgc.stm_is_enabled()
+        self.stm_blackholeinterps = {}
+
+    def get_or_create(self, builder):
+        assert rgc.stm_is_enabled()
+        # conflicts with new threads calling put_back
+
+        ident = rthread.get_ident()
+        interps = self.stm_blackholeinterps.get(ident, None)
+        if interps is not None and len(interps) > 0:
+            return interps.pop()
+        return BlackholeInterpreter(builder, 0)
+
+    def put_back(self, interp):
+        assert rgc.stm_is_enabled()
+
+        ident = rthread.get_ident()
+        interps = self.stm_blackholeinterps.get(ident, None)
+        if interps is None:
+            interps = self.stm_blackholeinterps.setdefault(ident, [])
+
+        interp.cleanup_registers()
+        interps.append(interp)
+
 
 class BlackholeInterpBuilder(object):
     verbose = True
@@ -51,6 +85,7 @@
         self.metainterp_sd = metainterp_sd
         self.num_interpreters = 0
         self.blackholeinterps = []
+        self.stm_cache = None
 
     def _cleanup_(self):
         # XXX don't assign a different list to blackholeinterp here,
@@ -233,19 +268,25 @@
         return handler
 
     def acquire_interp(self):
-        if rgc.stm_is_enabled():   # XXX for now, no caching
-            return BlackholeInterpreter(self, 0)
-        if len(self.blackholeinterps) > 0:
-            return self.blackholeinterps.pop()
+        if rgc.stm_is_enabled():
+            if self.stm_cache is None:
+                self.stm_cache = BlackholeInterpreterCache()
+            return self.stm_cache.get_or_create(self)
         else:
-            self.num_interpreters += 1
-            return BlackholeInterpreter(self, self.num_interpreters)
+            # no STM:
+            if len(self.blackholeinterps) > 0:
+                return self.blackholeinterps.pop()
+            else:
+                self.num_interpreters += 1
+                return BlackholeInterpreter(self, self.num_interpreters)
 
     def release_interp(self, interp):
         if rgc.stm_is_enabled():   # XXX for now, no caching
-            return
-        interp.cleanup_registers()
-        self.blackholeinterps.append(interp)
+            self.stm_cache.put_back(interp)
+        else:
+            # no STM:
+            interp.cleanup_registers()
+            self.blackholeinterps.append(interp)
 
 def check_shift_count(b):
     if not we_are_translated():

From noreply at buildbot.pypy.org  Mon Aug 24 10:27:20 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 10:27:20 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror: reverted some changes to get to the
 root of the problem
Message-ID: <20150824082720.7726A1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror
Changeset: r79179:a2ed66c15857
Date: 2015-08-24 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/a2ed66c15857/

Log:	reverted some changes to get to the root of the problem

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -536,7 +536,7 @@
             self.cpu.profile_agent.native_code_written(name,
                                                        rawstart, full_size)
         return AsmInfo(ops_offset, rawstart + looppos,
-                       size_excluding_failure_stuff - looppos, rawstart)
+                       size_excluding_failure_stuff - looppos)
 
     @rgc.no_release_gil
     def assemble_bridge(self, faildescr, inputargs, operations,
@@ -589,11 +589,12 @@
             name = "Bridge # %s" % (descr_number,)
             self.cpu.profile_agent.native_code_written(name,
                                                        rawstart, fullsize)
-        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
+        return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def stitch_bridge(self, faildescr, target):
-        assert target.rawstart != 0
-        self.patch_jump_for_descr(faildescr, target.rawstart)
+        if target is not None:
+            assert target.rawstart != 0
+            self.patch_jump_for_descr(faildescr, target.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -204,26 +204,27 @@
             vl.inputargs = version.inputargs
             vl.operations = version.operations
             vl.original_jitcell_token = jitcell_token
-            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+            send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                    faildescr, version.inputargs,
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
-            assert asminfo is not None
-            version._compiled = asminfo
-            faildescr.version = None
+            #assert asminfo is not None
+            #version._compiled = asminfo
+            #faildescr.version = None
+        loop.versions = None
         # stitch the rest of the traces
-        for lv in loop.versions:
-            if not lv.compiled():
-                # the version was never compiled, do not bother
-                # to assign it's fail descr
-                continue
-            for faildescr in lv.faildescrs:
-                assert isinstance(faildescr, CompileLoopVersionDescr)
-                version = faildescr.version
-                if version and version.compiled():
-                    cpu.stitch_bridge(faildescr, version._compiled)
-                faildescr.version = None
-    loop.versions = None
+        #for lv in loop.versions:
+        #    if not lv.compiled():
+        #        # the version was never compiled, do not bother
+        #        # to assign it's fail descr
+        #        continue
+        #    for faildescr in lv.faildescrs:
+        #        assert isinstance(faildescr, CompileLoopVersionDescr)
+        #        version = faildescr.version
+        #        if version and version.compiled():
+        #            cpu.stitch_bridge(faildescr, version._compiled)
+        #        faildescr.version = None
+    #loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
@@ -454,7 +455,7 @@
     #if metainterp_sd.warmrunnerdesc is not None:    # for tests
     #    metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(
     #        original_loop_token)
-    return asminfo
+    #return asminfo
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -809,6 +809,8 @@
         assert version.renamed_inputargs is not None
 
     def update_token(self, jitcell_token, all_target_tokens):
+        if self.compiled():
+            return
         # this is only invoked for versioned loops!
         label_index = index_of_first(rop.LABEL, self.operations, 0)
         label = self.operations[label_index]
diff --git a/rpython/jit/metainterp/logger.py b/rpython/jit/metainterp/logger.py
--- a/rpython/jit/metainterp/logger.py
+++ b/rpython/jit/metainterp/logger.py
@@ -102,7 +102,6 @@
         return descr.repr_of_descr()
 
     def repr_of_arg(self, arg):
-        return str(arg) # XXX
         try:
             mv = self.memo[arg]
         except KeyError:
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -277,9 +277,9 @@
                 if descr.loop_version():
                     root_version.register_guard(op, version)
 
-            if user_code:
-                version = loop.snapshot()
-                self.eliminate_array_bound_checks(loop, root_version, version)
+            #if user_code:
+            #    version = loop.snapshot()
+            #    self.eliminate_array_bound_checks(loop, root_version, version)
 
     def emit_operation(self, op):
         self.renamer.rename(op)
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1006,11 +1006,10 @@
     asmlen - assembler block length
     rawstart - address a guard can jump to
     """
-    def __init__(self, ops_offset, asmaddr, asmlen, rawstart):
+    def __init__(self, ops_offset, asmaddr, asmlen):
         self.ops_offset = ops_offset
         self.asmaddr = asmaddr
         self.asmlen = asmlen
-        self.rawstart = rawstart
 
 class JitDebugInfo(object):
     """ An object representing debug info. Attributes meanings:

From noreply at buildbot.pypy.org  Mon Aug 24 10:28:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 10:28:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: whack at tests a bit so
	they're runnable
Message-ID: <20150824082837.111961C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79180:3a31270fcdd4
Date: 2015-08-24 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/3a31270fcdd4/

Log:	whack at tests a bit so they're runnable

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -92,8 +92,7 @@
     def __init__(self, S, vtable, runner):
         self.S = S
         self._is_object = vtable is not None
-        self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
-                                    get_field_descr=LLGraphCPU.fielddescrof)
+        self.vtable = vtable
 
     def get_all_fielddescrs(self):
         return self.all_fielddescrs
@@ -103,7 +102,7 @@
 
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
-            self._corresponding_vtable))
+            self.vtable))
 
     def count_fields_if_immutable(self):
         return heaptracker.count_fields_if_immutable(self.S)
@@ -223,6 +222,8 @@
         self.fieldname = fieldname
         self.FIELD = getattr(A.OF, fieldname)
         self.arraydescr = runner.arraydescrof(A)
+        assert not heaptracker.has_gcstruct_a_vtable(A)
+        runner.sizeof(A.OF, None)
         self.fielddescr = runner.fielddescrof(A.OF, fieldname)
 
     def get_arraydescr(self):
@@ -414,6 +415,8 @@
         except KeyError:
             descr = SizeDescr(S, vtable, self)
             self.descrs[key] = descr
+            descr.all_fielddescrs = heaptracker.all_fielddescrs(self, S,
+                                    get_field_descr=LLGraphCPU.fielddescrof)
             return descr
 
     def fielddescrof(self, S, fieldname):
@@ -423,8 +426,8 @@
         except KeyError:
             descr = FieldDescr(S, fieldname)
             self.descrs[key] = descr
-            is_obj = heaptracker.has_gcstruct_a_vtable(S)
-            descr.parent_descr = self.sizeof(S, is_obj)
+            vtable = self.descrs[('size', S)].vtable
+            descr.parent_descr = self.sizeof(S, vtable)
             if self.vinfo_for_tests is not None:
                 descr.vinfo = self.vinfo_for_tests
             return descr
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -121,9 +121,9 @@
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
     #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
-    nodesize = cpu.sizeof(NODE, True)
-    nodesize2 = cpu.sizeof(NODE2, True)
-    nodesize3 = cpu.sizeof(NODE3, True)
+    nodesize = cpu.sizeof(NODE, node_vtable)
+    nodesize2 = cpu.sizeof(NODE2, node_vtable2)
+    nodesize3 = cpu.sizeof(NODE3, node_vtable3)
     valuedescr = cpu.fielddescrof(NODE, 'value')
     floatdescr = cpu.fielddescrof(NODE, 'floatval')
     chardescr = cpu.fielddescrof(NODE, 'charval')
@@ -139,7 +139,7 @@
     QUASI = lltype.GcStruct('QUASIIMMUT', ('inst_field', lltype.Signed),
                             ('mutate_field', rclass.OBJECTPTR),
                             hints={'immutable_fields': accessor})
-    quasisize = cpu.sizeof(QUASI, False)
+    quasisize = cpu.sizeof(QUASI, None)
     quasi = lltype.malloc(QUASI, immortal=True)
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
@@ -151,6 +151,7 @@
                                          ('ref', lltype.Ptr(OBJECT)))
     nodeobj = lltype.malloc(NODEOBJ)
     nodeobjvalue = lltype.cast_opaque_ptr(llmemory.GCREF, nodeobj)
+    cpu.sizeof(NODEOBJ, None)
     refdescr = cpu.fielddescrof(NODEOBJ, 'ref')
 
     INTOBJ_NOIMMUT = lltype.GcStruct('INTOBJ_NOIMMUT', ('parent', OBJECT),
@@ -160,15 +161,16 @@
                                             hints={'immutable': True})
     intobj_noimmut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
+    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, intobj_noimmut_vtable)
+    immut_descr = cpu.sizeof(INTOBJ_IMMUT, intobj_immut_vtable)
     noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval')
     immut_intval = cpu.fielddescrof(INTOBJ_IMMUT, 'intval')
-    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, True)
-    immut_descr = cpu.sizeof(INTOBJ_IMMUT, True)
 
     PTROBJ_IMMUT = lltype.GcStruct('PTROBJ_IMMUT', ('parent', OBJECT),
                                             ('ptrval', lltype.Ptr(OBJECT)),
                                             hints={'immutable': True})
     ptrobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
+    cpu.sizeof(PTROBJ_IMMUT, None)
     immut_ptrval = cpu.fielddescrof(PTROBJ_IMMUT, 'ptrval')
 
     arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))
@@ -178,7 +180,7 @@
     # a GcStruct not inheriting from OBJECT
     S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed),
                         ('b', lltype.Ptr(NODE)))
-    ssize = cpu.sizeof(S, False)
+    ssize = cpu.sizeof(S, None)
     adescr = cpu.fielddescrof(S, 'a')
     abisdescr = cpu.fielddescrof(S, 'abis')
     bdescr = cpu.fielddescrof(S, 'b')
@@ -193,15 +195,16 @@
         ('inst_w_seq', llmemory.GCREF), ('inst_index', lltype.Signed),
         ('inst_w_list', llmemory.GCREF), ('inst_length', lltype.Signed),
         ('inst_start', lltype.Signed), ('inst_step', lltype.Signed))
+    w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
+    cpu.sizeof(W_ROOT, w_root_vtable)
     inst_w_seq = cpu.fielddescrof(W_ROOT, 'inst_w_seq')
     inst_index = cpu.fielddescrof(W_ROOT, 'inst_index')
     inst_length = cpu.fielddescrof(W_ROOT, 'inst_length')
     inst_start = cpu.fielddescrof(W_ROOT, 'inst_start')
     inst_step = cpu.fielddescrof(W_ROOT, 'inst_step')
     inst_w_list = cpu.fielddescrof(W_ROOT, 'inst_w_list')
-    w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     
-    tsize = cpu.sizeof(T, False)
+    tsize = cpu.sizeof(T, None)
     cdescr = cpu.fielddescrof(T, 'c')
     ddescr = cpu.fielddescrof(T, 'd')
     arraydescr3 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(NODE)))
@@ -214,10 +217,10 @@
     SIMPLE = lltype.GcStruct('simple',
         ('parent', OBJECT),
         ('value', lltype.Signed))
-    simpledescr = cpu.sizeof(SIMPLE, True)
+    simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
+    simpledescr = cpu.sizeof(SIMPLE, simple_vtable)
     simplevalue = cpu.fielddescrof(SIMPLE, 'value')
-    simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    usize = cpu.sizeof(U, True)
+    usize = cpu.sizeof(U, u_vtable)
     onedescr = cpu.fielddescrof(U, 'one')
 
     FUNC = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -349,7 +352,7 @@
 
     jit_virtual_ref_vtable = vrefinfo.jit_virtual_ref_vtable
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
-    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
+    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, None)
 
     namespace = locals()
 

From noreply at buildbot.pypy.org  Mon Aug 24 10:28:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 10:28:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: Backed out changeset
	3a31270fcdd4
Message-ID: <20150824082839.1EBC71C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79181:180f01789a16
Date: 2015-08-24 10:26 +0200
http://bitbucket.org/pypy/pypy/changeset/180f01789a16/

Log:	Backed out changeset 3a31270fcdd4

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -92,7 +92,8 @@
     def __init__(self, S, vtable, runner):
         self.S = S
         self._is_object = vtable is not None
-        self.vtable = vtable
+        self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
+                                    get_field_descr=LLGraphCPU.fielddescrof)
 
     def get_all_fielddescrs(self):
         return self.all_fielddescrs
@@ -102,7 +103,7 @@
 
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
-            self.vtable))
+            self._corresponding_vtable))
 
     def count_fields_if_immutable(self):
         return heaptracker.count_fields_if_immutable(self.S)
@@ -222,8 +223,6 @@
         self.fieldname = fieldname
         self.FIELD = getattr(A.OF, fieldname)
         self.arraydescr = runner.arraydescrof(A)
-        assert not heaptracker.has_gcstruct_a_vtable(A)
-        runner.sizeof(A.OF, None)
         self.fielddescr = runner.fielddescrof(A.OF, fieldname)
 
     def get_arraydescr(self):
@@ -415,8 +414,6 @@
         except KeyError:
             descr = SizeDescr(S, vtable, self)
             self.descrs[key] = descr
-            descr.all_fielddescrs = heaptracker.all_fielddescrs(self, S,
-                                    get_field_descr=LLGraphCPU.fielddescrof)
             return descr
 
     def fielddescrof(self, S, fieldname):
@@ -426,8 +423,8 @@
         except KeyError:
             descr = FieldDescr(S, fieldname)
             self.descrs[key] = descr
-            vtable = self.descrs[('size', S)].vtable
-            descr.parent_descr = self.sizeof(S, vtable)
+            is_obj = heaptracker.has_gcstruct_a_vtable(S)
+            descr.parent_descr = self.sizeof(S, is_obj)
             if self.vinfo_for_tests is not None:
                 descr.vinfo = self.vinfo_for_tests
             return descr
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -121,9 +121,9 @@
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
     #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
-    nodesize = cpu.sizeof(NODE, node_vtable)
-    nodesize2 = cpu.sizeof(NODE2, node_vtable2)
-    nodesize3 = cpu.sizeof(NODE3, node_vtable3)
+    nodesize = cpu.sizeof(NODE, True)
+    nodesize2 = cpu.sizeof(NODE2, True)
+    nodesize3 = cpu.sizeof(NODE3, True)
     valuedescr = cpu.fielddescrof(NODE, 'value')
     floatdescr = cpu.fielddescrof(NODE, 'floatval')
     chardescr = cpu.fielddescrof(NODE, 'charval')
@@ -139,7 +139,7 @@
     QUASI = lltype.GcStruct('QUASIIMMUT', ('inst_field', lltype.Signed),
                             ('mutate_field', rclass.OBJECTPTR),
                             hints={'immutable_fields': accessor})
-    quasisize = cpu.sizeof(QUASI, None)
+    quasisize = cpu.sizeof(QUASI, False)
     quasi = lltype.malloc(QUASI, immortal=True)
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
@@ -151,7 +151,6 @@
                                          ('ref', lltype.Ptr(OBJECT)))
     nodeobj = lltype.malloc(NODEOBJ)
     nodeobjvalue = lltype.cast_opaque_ptr(llmemory.GCREF, nodeobj)
-    cpu.sizeof(NODEOBJ, None)
     refdescr = cpu.fielddescrof(NODEOBJ, 'ref')
 
     INTOBJ_NOIMMUT = lltype.GcStruct('INTOBJ_NOIMMUT', ('parent', OBJECT),
@@ -161,16 +160,15 @@
                                             hints={'immutable': True})
     intobj_noimmut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, intobj_noimmut_vtable)
-    immut_descr = cpu.sizeof(INTOBJ_IMMUT, intobj_immut_vtable)
     noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval')
     immut_intval = cpu.fielddescrof(INTOBJ_IMMUT, 'intval')
+    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, True)
+    immut_descr = cpu.sizeof(INTOBJ_IMMUT, True)
 
     PTROBJ_IMMUT = lltype.GcStruct('PTROBJ_IMMUT', ('parent', OBJECT),
                                             ('ptrval', lltype.Ptr(OBJECT)),
                                             hints={'immutable': True})
     ptrobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    cpu.sizeof(PTROBJ_IMMUT, None)
     immut_ptrval = cpu.fielddescrof(PTROBJ_IMMUT, 'ptrval')
 
     arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))
@@ -180,7 +178,7 @@
     # a GcStruct not inheriting from OBJECT
     S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed),
                         ('b', lltype.Ptr(NODE)))
-    ssize = cpu.sizeof(S, None)
+    ssize = cpu.sizeof(S, False)
     adescr = cpu.fielddescrof(S, 'a')
     abisdescr = cpu.fielddescrof(S, 'abis')
     bdescr = cpu.fielddescrof(S, 'b')
@@ -195,16 +193,15 @@
         ('inst_w_seq', llmemory.GCREF), ('inst_index', lltype.Signed),
         ('inst_w_list', llmemory.GCREF), ('inst_length', lltype.Signed),
         ('inst_start', lltype.Signed), ('inst_step', lltype.Signed))
-    w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    cpu.sizeof(W_ROOT, w_root_vtable)
     inst_w_seq = cpu.fielddescrof(W_ROOT, 'inst_w_seq')
     inst_index = cpu.fielddescrof(W_ROOT, 'inst_index')
     inst_length = cpu.fielddescrof(W_ROOT, 'inst_length')
     inst_start = cpu.fielddescrof(W_ROOT, 'inst_start')
     inst_step = cpu.fielddescrof(W_ROOT, 'inst_step')
     inst_w_list = cpu.fielddescrof(W_ROOT, 'inst_w_list')
+    w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     
-    tsize = cpu.sizeof(T, None)
+    tsize = cpu.sizeof(T, False)
     cdescr = cpu.fielddescrof(T, 'c')
     ddescr = cpu.fielddescrof(T, 'd')
     arraydescr3 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(NODE)))
@@ -217,10 +214,10 @@
     SIMPLE = lltype.GcStruct('simple',
         ('parent', OBJECT),
         ('value', lltype.Signed))
+    simpledescr = cpu.sizeof(SIMPLE, True)
+    simplevalue = cpu.fielddescrof(SIMPLE, 'value')
     simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    simpledescr = cpu.sizeof(SIMPLE, simple_vtable)
-    simplevalue = cpu.fielddescrof(SIMPLE, 'value')
-    usize = cpu.sizeof(U, u_vtable)
+    usize = cpu.sizeof(U, True)
     onedescr = cpu.fielddescrof(U, 'one')
 
     FUNC = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -352,7 +349,7 @@
 
     jit_virtual_ref_vtable = vrefinfo.jit_virtual_ref_vtable
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
-    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, None)
+    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
 
     namespace = locals()
 

From noreply at buildbot.pypy.org  Mon Aug 24 10:28:41 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 10:28:41 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: bakcout
Message-ID: <20150824082841.52B3D1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79182:40b6f6640e9e
Date: 2015-08-24 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/40b6f6640e9e/

Log:	bakcout

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -101,6 +101,9 @@
     def is_object(self):
         return self._is_object
 
+    def as_vtable_size_descr(self):
+        return self
+
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
             self._corresponding_vtable))
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -59,6 +59,9 @@
         return False
 
 class SizeDescrWithVTable(SizeDescr):
+    def as_vtable_size_descr(self):
+        return self
+
     def is_object(self):
         return True
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -121,8 +121,7 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- GETFIELD_GC ----------
-            if op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F,
-                                 rop.GETFIELD_GC_R):
+            if op.getopnum() == rop.GETFIELD_GC:
                 self.handle_getfield_gc(op)
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
@@ -195,7 +194,7 @@
         followed by a bunch of 'setfields', and the 'pending_zeros'
         optimization we do here is meant for this case."""
         self.emit_pending_zeros()
-        self.emit_op(op)
+        self.newops.append(op)
 
     # ----------
 
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -8,6 +8,7 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
+from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, lltype, llmemory
 from rpython.rtyper import rclass
@@ -51,6 +52,7 @@
         O = lltype.GcStruct('O', ('parent', rclass.OBJECT),
                                  ('x', lltype.Signed))
         o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
+        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -7,6 +7,7 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
+from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -63,6 +64,7 @@
                                  ('x', lltype.Signed))
         o_descr = self.cpu.sizeof(O, True)
         o_vtable = globals()['o_vtable']
+        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1950,7 +1950,8 @@
         cpu = self.cpu
         t_box, T_box, descr = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.getint()), rclass.CLASSTYPE)
+            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
+        heaptracker.register_known_gctype(cpu, vtable, self.T)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         assert r1 != r2
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -123,6 +123,8 @@
         vtable.name = rclass.alloc_array_name(name)
         self.structure_types_and_vtables.append((S, vtable))
         #
+        heaptracker.register_known_gctype(self.cpu, vtable, S)
+        #
         return S, vtable
 
     def get_random_structure(self, r, has_vtable=False):
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -74,6 +74,7 @@
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
         self.assembler.finished(self.callcontrol.callinfocollection)
+        heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -88,6 +88,57 @@
 
 # ____________________________________________________________
 
+VTABLETYPE = rclass.CLASSTYPE
+
+def register_known_gctype(cpu, vtable, STRUCT):
+    # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
+    sizedescr = cpu.sizeof(STRUCT, vtable)
+    assert sizedescr.as_vtable_size_descr() is sizedescr
+    if getattr(sizedescr, '_corresponding_vtable', None):
+        assert sizedescr._corresponding_vtable == vtable
+        return
+    assert lltype.typeOf(vtable) == VTABLETYPE
+    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
+        cpu.tracker._all_size_descrs_with_vtable = []
+        cpu.tracker._vtable_to_descr_dict = None
+    cpu.tracker._all_size_descrs_with_vtable.append(sizedescr)
+    sizedescr._corresponding_vtable = vtable
+
+def finish_registering(cpu):
+    # annotation hack for small examples which have no vtable at all
+    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
+    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
+    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
+    pass
+
+def vtable2descr(cpu, vtable):
+    assert lltype.typeOf(vtable) is lltype.Signed
+    vtable = int2adr(vtable)
+    if we_are_translated():
+        # Build the dict {vtable: sizedescr} at runtime.
+        # This is necessary because the 'vtables' are just pointers to
+        # static data, so they can't be used as keys in prebuilt dicts.
+        d = cpu.tracker._vtable_to_descr_dict
+        if d is None:
+            d = cpu.tracker._vtable_to_descr_dict = {}
+            for descr in cpu.tracker._all_size_descrs_with_vtable:
+                key = descr._corresponding_vtable
+                key = llmemory.cast_ptr_to_adr(key)
+                d[key] = descr
+        return d[vtable]
+    else:
+        vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE)
+        for descr in cpu.tracker._all_size_descrs_with_vtable:
+            if descr._corresponding_vtable == vtable:
+                return descr
+        raise KeyError(vtable)
+
+def descr2vtable(cpu, descr):
+    from rpython.jit.metainterp import history
+    assert isinstance(descr, history.AbstractDescr)
+    vtable = descr.as_vtable_size_descr()._corresponding_vtable
+    vtable = llmemory.cast_ptr_to_adr(vtable)
+    return adr2int(vtable)
 
 def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None,
                     get_field_descr=None):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -932,6 +932,7 @@
                     return self._do_builtin_call(op, 'alloc_with_del', [],
                                                  extra=(RESULT, vtable),
                                                  extrakey=STRUCT)
+            heaptracker.register_known_gctype(self.cpu, vtable, STRUCT)
             opname = 'new_with_vtable'
         else:
             opname = 'new'
@@ -1935,6 +1936,10 @@
     # VirtualRefs.
 
     def _handle_virtual_ref_call(self, op, oopspec_name, args):
+        vrefinfo = self.callcontrol.virtualref_info
+        heaptracker.register_known_gctype(self.cpu,
+                                          vrefinfo.jit_virtual_ref_vtable,
+                                          vrefinfo.JIT_VIRTUAL_REF)
         return SpaceOperation(oopspec_name, list(args), op.result)
 
     # -----------
diff --git a/rpython/jit/codewriter/test/test_codewriter.py b/rpython/jit/codewriter/test/test_codewriter.py
--- a/rpython/jit/codewriter/test/test_codewriter.py
+++ b/rpython/jit/codewriter/test/test_codewriter.py
@@ -25,6 +25,8 @@
 class FakeSizeDescr(AbstractDescr):
     def __init__(self, STRUCT):
         self.STRUCT = STRUCT
+    def as_vtable_size_descr(self):
+        return self
 
 class FakeArrayDescr(AbstractDescr):
     def __init__(self, ARRAY):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -39,6 +39,8 @@
         self.oopspecindex = oopspecindex
     def __repr__(self):
         return ''
+    def as_vtable_size_descr(self):
+        return self
 
 class FakeDict(object):
     def __getitem__(self, key):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -50,7 +50,8 @@
         return FakeDescr(('sizedescr', STRUCT))
 
 class FakeDescr(tuple):
-    pass
+    def as_vtable_size_descr(self):
+        return self
 
 class FakeLink:
     args = []
@@ -568,6 +569,9 @@
     op1 = Transformer(cpu).rewrite_operation(op)
     assert op1.opname == 'new_with_vtable'
     assert op1.args == [('sizedescr', S)]
+    #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check]
+    vtable_int = heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtable))
+    assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0]
 
 def test_malloc_new_with_destructor():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -242,11 +242,6 @@
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
 
-    def _get_buffer(self):
-        buffer = self.buffer
-        assert buffer is not None
-        return buffer
-
     def getitem_raw(self, offset, itemsize, descr):
         if not self.is_virtual():
             raise InvalidRawOperation
@@ -254,39 +249,37 @@
             # for the test above: it's not enough to check is_virtual()
             # on the original object, because it might be a VRawSliceValue
             # instead.  If it is a virtual one, then we'll reach here anway.
-        return self._get_buffer().read_value(offset, itemsize, descr)
+        return self.buffer.read_value(offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         if not self.is_virtual():
             raise InvalidRawOperation
-        self._get_buffer().write_value(offset, itemsize, descr, itemop)
+        self.buffer.write_value(offset, itemsize, descr, itemop)
 
     def is_virtual(self):
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
         self.size = -1
-        buffer = self._get_buffer()
-        for i in range(len(buffer.offsets)):
+        for i in range(len(self.buffer.offsets)):
             # write the value
-            offset = buffer.offsets[i]
-            descr = buffer.descrs[i]
-            itembox = buffer.values[i]
+            offset = self.buffer.offsets[i]
+            descr = self.buffer.descrs[i]
+            itembox = self.buffer.values[i]
             setfield_op = ResOperation(rop.RAW_STORE,
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(setfield_op)
 
     def _visitor_walk_recursive(self, op, visitor, optimizer):
-        itemboxes = self._get_buffer().values
+        itemboxes = self.buffer.values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
-        buffer = self._get_buffer()
         return visitor.visit_vrawbuffer(self.size,
-                                        buffer.offsets[:],
-                                        buffer.descrs[:])
+                                        self.buffer.offsets[:],
+                                        self.buffer.descrs[:])
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -12,6 +12,7 @@
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.logger import LogOperations
+from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import OpParser
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
@@ -351,6 +352,17 @@
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
     vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
 
+    register_known_gctype(cpu, node_vtable,  NODE)
+    register_known_gctype(cpu, node_vtable2, NODE2)
+    register_known_gctype(cpu, node_vtable3, NODE3)
+    register_known_gctype(cpu, u_vtable,     U)
+    register_known_gctype(cpu, simple_vtable,     SIMPLE)
+    register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
+    register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
+    register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
+    register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
+    register_known_gctype(cpu, w_root_vtable, W_ROOT)
+
     namespace = locals()
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,4 +1,5 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.codewriter.heaptracker import descr2vtable
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
@@ -198,7 +199,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        known_class = ConstInt(op.getdescr().get_vtable())
+        known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr()))
         self.make_virtual(known_class, op, op.getdescr())
 
     def optimize_NEW(self, op):
diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -597,7 +597,8 @@
                 self._no = no
 
         class FakeDescr:
-            pass
+            def as_vtable_size_descr(self):
+                return self
 
         class FakeCPU(object):
             supports_floats = False

From noreply at buildbot.pypy.org  Mon Aug 24 10:31:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 10:31:24 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix one test
Message-ID: <20150824083124.909CC1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79183:df8324a1d7b4
Date: 2015-08-24 10:31 +0200
http://bitbucket.org/pypy/pypy/changeset/df8324a1d7b4/

Log:	fix one test

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7613,7 +7613,7 @@
         [p0]
         p1 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p1) []
-        i1 = arraylen_gc(p1)
+        i1 = arraylen_gc(p1, descr=gcarraydescr)
         i2 = int_ge(i1, 8)
         guard_true(i2) []
         p2 = getarrayitem_gc_r(p1, 7, descr=gcarraydescr)
@@ -7622,7 +7622,7 @@
         expected = """
         [p0, p1, p2]
         call_n(p2, descr=nonwritedescr)
-        i3 = arraylen_gc(p1) # Should be killed by backend
+        i3 = arraylen_gc(p1, descr=gcarraydescr) # Should be killed by backend
         jump(p0, p1, p2)
         """
         self.optimize_loop(ops, expected, expected_short=short)

From noreply at buildbot.pypy.org  Mon Aug 24 11:06:11 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 11:06:11 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror: some more changes
Message-ID: <20150824090611.4902B1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror
Changeset: r79184:e967b7cba547
Date: 2015-08-24 10:51 +0200
http://bitbucket.org/pypy/pypy/changeset/e967b7cba547/

Log:	some more changes

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -532,7 +532,7 @@
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
                'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
-               'status')
+               'status', 'version')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -543,6 +543,7 @@
     rd_accum_list = None
 
     status = r_uint(0)
+    version = None
 
     def copy_all_attributes_from(self, other):
         assert isinstance(other, ResumeGuardDescr)
@@ -747,11 +748,6 @@
 class CompileLoopVersionDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_EARLY_EXIT
 
-    operations = None
-    inputargs = None
-    faillocs = None
-    version = None
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         assert 0, "this guard must never fail"
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -893,6 +893,7 @@
         #
         self.operations = self.operations[:-1] + loop.operations
         self.versions = loop.versions
+        loop.versions = None
         if loop.quasi_immutable_deps:
             self.quasi_immutable_deps.update(loop.quasi_immutable_deps)
 
@@ -910,10 +911,6 @@
         return None
 
     def snapshot(self):
-        if len(self.versions) == 0:
-            # create a root version, simplyfies the code in compile.py
-            self.versions.append(LoopVersion(None))
-        root_version = self.versions[0]
         version = LoopVersion(self)
         self.versions.append(version)
         return version
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -42,7 +42,9 @@
     if user_code and user_loop_bail_fast_path(loop, warmstate):
         return
     # the original loop (output of optimize_unroll)
+    loop.versions.append(LoopVersion(None))
     version = loop.snapshot()
+    old_loop_len = len(loop.operations)
     try:
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
@@ -65,7 +67,7 @@
         #
         nano = int((end-start)*10.0**9)
         debug_print("# vecopt factor: %d opcount: (%d -> %d) took %dns" % \
-                      (opt.unroll_count+1, len(version.operations), len(loop.operations), nano))
+                      (opt.unroll_count+1, old_loop_len, len(loop.operations), nano))
         debug_stop("vec-opt-loop")
         #
     except NotAVectorizeableLoop:
@@ -87,8 +89,7 @@
             from rpython.rtyper.lltypesystem import lltype
             from rpython.rtyper.lltypesystem.lloperation import llop
             llop.debug_print_traceback(lltype.Void)
-        else:
-            raise
+        raise e
 
 def user_loop_bail_fast_path(loop, warmstate):
     """ In a fast path over the trace loop: try to prevent vecopt
@@ -628,7 +629,8 @@
         descr = None
         guard_true_false = tgt_op.getopnum() in (rop.GUARD_TRUE, rop.GUARD_FALSE)
         if guard_true_false:
-            descr = CompileLoopVersionDescr()
+            #descr = CompileLoopVersionDescr()
+            descr = ResumeAtLoopHeaderDescr()
         else:
             descr = ResumeAtLoopHeaderDescr()
         if olddescr:

From noreply at buildbot.pypy.org  Mon Aug 24 11:06:13 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 11:06:13 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror2: start from the other way arround
 and apply changes until it does not work anymore
Message-ID: <20150824090613.9E5411C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror2
Changeset: r79185:0e7fdd3779b2
Date: 2015-08-24 11:05 +0200
http://bitbucket.org/pypy/pypy/changeset/0e7fdd3779b2/

Log:	start from the other way arround and apply changes until it does not
	work anymore

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -199,10 +199,11 @@
             vl.inputargs = version.inputargs
             vl.operations = version.operations
             vl.original_jitcell_token = jitcell_token
-            send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
                                    faildescr, version.inputargs,
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
+            version.compiled = asminfo
             for faildescr in version.faildescrs[1:]:
                 cpu.stitch_bridge(faildescr, jitcell_token)
     loop.versions = None
@@ -436,6 +437,7 @@
     #if metainterp_sd.warmrunnerdesc is not None:    # for tests
     #    metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(
     #        original_loop_token)
+    return asminfo
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -52,7 +52,7 @@
         gso = GuardStrengthenOpt(opt.dependency_graph.index_vars)
         gso.propagate_all_forward(opt.loop, user_code)
         # connect all compile loop version fail descriptors to this version
-        version.register_all_guards(loop.operations, opt.appended_arg_count)
+        #version.register_all_guards(loop.operations, opt.appended_arg_count)
         #
         #
         end = time.clock()

From noreply at buildbot.pypy.org  Mon Aug 24 11:43:09 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 11:43:09 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: changes to translate this version
Message-ID: <20150824094309.7AF221C0400@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79186:8b191cded36c
Date: 2015-08-24 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/8b191cded36c/

Log:	changes to translate this version

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -592,8 +592,9 @@
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
 
     def stitch_bridge(self, faildescr, target):
-        assert target != 0
-        self.patch_jump_for_descr(faildescr, target.rawstart)
+        pass
+        #assert target != 0
+        #self.patch_jump_for_descr(faildescr, target.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -211,6 +211,7 @@
         # stitch the rest of the traces
         for version in loop.versions:
             for faildescr in version.faildescrs[1:]:
+                assert isinstance(faildescr, CompileLoopVersionDescr)
                 if faildescr.version.compiled:
                     cpu.stitch_bridge(faildescr, version.compiled)
     loop.versions = None

From noreply at buildbot.pypy.org  Mon Aug 24 12:02:25 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 12:02:25 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: finished the refactoring. the root
 loop has now it's own version,
 which makes generate_pending_loop_versions easier to understand. a fail
 descr now always points to the loop version (field version) it would like
 to jump to
Message-ID: <20150824100225.BFC4F1C1155@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79187:743dc64b5f8a
Date: 2015-08-21 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/743dc64b5f8a/

Log:	finished the refactoring. the root loop has now it's own version,
	which makes generate_pending_loop_versions easier to understand. a
	fail descr now always points to the loop version (field version) it
	would like to jump to handle not compiled versions (because they are
	not needed) gracefully

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -162,7 +162,7 @@
 
     if loop.versions is not None:
         # every different loop version must update their target tokens
-        for version in loop.versions:
+        for version in loop.versions[1:]:
             version.update_token(jitcell_token, all_target_tokens)
 
     if not loop.quasi_immutable_deps:
@@ -194,10 +194,11 @@
     if loop.versions is not None:
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
-        for version in loop.versions:
-            if len(version.faildescrs) == 0:
+        root = loop.versions[0]
+        for faildescr in root.faildescrs:
+            version = faildescr.version
+            if not version or version.compiled():
                 continue
-            faildescr = version.faildescrs[0]
             vl = create_empty_loop(metainterp)
             vl.inputargs = version.inputargs
             vl.operations = version.operations
@@ -207,13 +208,19 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version.compiled = asminfo
+            version._compiled = asminfo
+            faildescr.version = None
         # stitch the rest of the traces
-        for version in loop.versions:
-            for faildescr in version.faildescrs[1:]:
-                assert isinstance(faildescr, CompileLoopVersionDescr)
-                if faildescr.version.compiled:
-                    cpu.stitch_bridge(faildescr, version.compiled)
+        for lv in loop.versions:
+            if not lv.compiled():
+                # the version was never compiled, do not bother
+                # to assign it's fail descr
+                continue
+            for faildescr in lv.faildescrs:
+                version = faildescr.version
+                if version and version.compiled():
+                    cpu.stitch_bridge(faildescr, version._compiled)
+                faildescr.version = None
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -748,28 +748,62 @@
 
 class LoopVersion(object):
 
-    def __init__(self, operations):
-        self.operations = operations
+    def __init__(self, loop):
         self.faildescrs = []
-        #
-        idx = index_of_first(rop.LABEL, operations)
-        assert idx >= 0
-        label = operations[idx]
-        self.label_pos = idx
-        self.inputargs = label.getarglist()
-        self.renamed_inputargs = label.getarglist()
-        self.compiled = None
+        self._compiled = None
+        if loop:
+            self.operations = self.copy_operations(loop.operations) 
+            idx = index_of_first(rop.LABEL, self.operations)
+            assert idx >= 0
+            label = self.operations[idx]
+            self.inputargs = label.getarglist()
+            self.renamed_inputargs = label.getarglist()
+        else:
+            self.operations = None
+            self.inputargs = None
+            self.renamed_inputargs = None
 
-    def register_guard(self, op):
+    def compiled(self):
+        if self.operations is None:
+            # root version must always be compiled
+            return True
+
+        return self._compiled is not None
+
+    def copy_operations(self, operations):
+        from rpython.jit.metainterp.compile import ResumeGuardDescr
+        ignore = (rop.DEBUG_MERGE_POINT,)
+        oplist = []
+        for op in operations:
+            if op.getopnum() in ignore:
+                continue
+            cloned = op.clone()
+            oplist.append(cloned)
+            if cloned.is_guard():
+                olddescr = cloned.getdescr()
+                if not olddescr:
+                    continue
+                descr = olddescr.clone()
+                cloned.setdescr(descr)
+                if olddescr.loop_version():
+                    # copy the version
+                    descr.version = olddescr.version
+                    self.faildescrs.append(descr)
+        return oplist
+
+    def register_guard(self, op, version):
         from rpython.jit.metainterp.compile import CompileLoopVersionDescr
         assert isinstance(op, GuardResOp)
         descr = op.getdescr()
+        if not descr.loop_version():
+            assert 0, "cannot register a guard that is not a CompileLoopVersionDescr"
         assert isinstance(descr, CompileLoopVersionDescr)
+        descr.version = version
         self.faildescrs.append(descr)
-        descr.version = self
         # note: stitching a guard must resemble the order of the label
         # otherwise a wrong mapping is handed to the register allocator
-        op.setfailargs(self.renamed_inputargs)
+        op.setfailargs(version.renamed_inputargs)
+        assert version.renamed_inputargs is not None
 
     def update_token(self, jitcell_token, all_target_tokens):
         # this is only invoked for versioned loops!
@@ -798,7 +832,6 @@
         label.setdescr(token)
         jump.setdescr(token)
 
-
 class TreeLoop(object):
     inputargs = None
     operations = None
@@ -872,9 +905,11 @@
         return None
 
     def snapshot(self):
-        faildescrs = []
-        version = LoopVersion(self.copy_operations(faildescrs))
-        version.faildescrs = faildescrs
+        if len(self.versions) == 0:
+            # create a root version, simplyfies the code in compile.py
+            self.versions.append(LoopVersion(None))
+        root_version = self.versions[0]
+        version = LoopVersion(self)
         if not we_are_translated():
             print "LOOP SNAPSHOT"
             for op in version.operations:
@@ -882,23 +917,6 @@
         self.versions.append(version)
         return version
 
-    def copy_operations(self, faildescrs=None):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
-        ignore = (rop.DEBUG_MERGE_POINT,)
-        operations = []
-        for op in self.operations:
-            if op.getopnum() in ignore:
-                continue
-            cloned = op.clone()
-            operations.append(cloned)
-            descr = cloned.getdescr()
-            if cloned.is_guard() and descr:
-                assert isinstance(descr, ResumeGuardDescr)
-                cloned.setdescr(descr.clone())
-                if faildescrs and descr.loop_version():
-                    faildescrs.append(cloned.getdescr())
-        return operations
-
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
 
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -152,8 +152,7 @@
         assert operations[self.index] is self.op
         operations[self.index] = None
         descr = self.op.getdescr()
-        if isinstance(descr, CompileLoopVersionDescr) and descr.version:
-            descr.version.faildescrs.remove(descr)
+        if descr and descr.loop_version():
             descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None
@@ -265,19 +264,20 @@
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
         #
-        assert len(loop.versions) == 1, "none or more than one version created"
-        version = loop.versions[0]
+        assert len(loop.versions) == 2, "need one orignal loop version (before trans) and the current one"
+        root_version = loop.versions[0]
+        version = loop.versions[1]
 
         for op in loop.operations:
             if not op.is_guard():
                 continue
             descr = op.getdescr()
             if descr.loop_version():
-                version.register_guard(op)
+                root_version.register_guard(op, version)
 
         if user_code:
             version = loop.snapshot()
-            self.eliminate_array_bound_checks(loop, version)
+            self.eliminate_array_bound_checks(loop, root_version, version)
 
     def emit_operation(self, op):
         self.renamer.rename(op)
@@ -286,7 +286,7 @@
     def operation_position(self):
         return len(self._newoperations)
 
-    def eliminate_array_bound_checks(self, loop, version):
+    def eliminate_array_bound_checks(self, loop, root_version, version):
         self._newoperations = []
         for key, guards in self.strongest_guards.items():
             if len(guards) <= 1:
@@ -299,7 +299,7 @@
                 transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
                     other.set_to_none(loop.operations)
-                    version.register_guard(transitive_guard)
+                    root_version.register_guard(transitive_guard, version)
 
         if self.has_two_labels:
             oplist = [loop.operations[0]] + self._newoperations + \
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -38,6 +38,7 @@
     user_code = not jitdriver_sd.vec and warmstate.vec_all
     if user_code and user_loop_bail_fast_path(loop, warmstate):
         return
+    # the original loop (output of optimize_unroll)
     version = loop.snapshot()
     try:
         debug_start("vec-opt-loop")
@@ -493,7 +494,7 @@
             return
         if vector:
             # add accumulation info to the descriptor
-            for version in self.loop.versions:
+            for version in self.loop.versions[1:]:
                 # this needs to be done for renamed (accum arguments)
                 version.renamed_inputargs = [ renamer.rename_map.get(arg,arg) for arg in version.inputargs ]
             self.appended_arg_count = len(sched_data.invariant_vector_vars)
@@ -874,7 +875,7 @@
                 box = result
             elif accum.operator == Accum.MULTIPLY:
                 # multiply is only supported by floats
-                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstFloat(1.0)], box)
+                op = ResOperation(rop.VEC_FLOAT_EXPAND, [ConstFloat(1.0), ConstInt(size)], box)
                 sched_data.invariant_oplist.append(op)
             else:
                 raise NotImplementedError("can only handle + and *")

From noreply at buildbot.pypy.org  Mon Aug 24 12:02:27 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 12:02:27 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: translation issues
Message-ID: <20150824100227.DBAB21C1155@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79188:e119ce67b48b
Date: 2015-08-24 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/e119ce67b48b/

Log:	translation issues

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -529,7 +529,7 @@
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
                'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
-               'status')
+               'status', 'version')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -540,6 +540,7 @@
     rd_accum_list = None
 
     status = r_uint(0)
+    version = None
 
     def copy_all_attributes_from(self, other):
         assert isinstance(other, ResumeGuardDescr)
@@ -744,11 +745,6 @@
 class CompileLoopVersionDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_EARLY_EXIT
 
-    operations = None
-    inputargs = None
-    faillocs = None
-    version = None
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         assert 0, "this guard must never fail"
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -771,7 +771,7 @@
         return self._compiled is not None
 
     def copy_operations(self, operations):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr
+        from rpython.jit.metainterp.compile import ResumeGuardDescr, CompileLoopVersionDescr 
         ignore = (rop.DEBUG_MERGE_POINT,)
         oplist = []
         for op in operations:
@@ -787,6 +787,8 @@
                 cloned.setdescr(descr)
                 if olddescr.loop_version():
                     # copy the version
+                    assert isinstance(descr, CompileLoopVersionDescr)
+                    assert isinstance(olddescr, CompileLoopVersionDescr)
                     descr.version = olddescr.version
                     self.faildescrs.append(descr)
         return oplist
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -153,6 +153,8 @@
         operations[self.index] = None
         descr = self.op.getdescr()
         if descr and descr.loop_version():
+            from rpython.jit.metainterp.compile import CompileLoopVersionDescr
+            assert isinstance(descr, CompileLoopVersionDescr)
             descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None

From noreply at buildbot.pypy.org  Mon Aug 24 12:14:02 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 12:14:02 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: LoopVersion _attrs_ added,
 renamed _compiled to compile_data
Message-ID: <20150824101402.1D86E1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79189:e679d3e0131f
Date: 2015-08-24 12:14 +0200
http://bitbucket.org/pypy/pypy/changeset/e679d3e0131f/

Log:	LoopVersion _attrs_ added, renamed _compiled to compile_data

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -208,7 +208,7 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version._compiled = asminfo
+            version.compile_data = asminfo
             faildescr.version = None
         # stitch the rest of the traces
         for lv in loop.versions:
@@ -219,7 +219,7 @@
             for faildescr in lv.faildescrs:
                 version = faildescr.version
                 if version and version.compiled():
-                    cpu.stitch_bridge(faildescr, version._compiled)
+                    cpu.stitch_bridge(faildescr, version.compile_data)
                 faildescr.version = None
     loop.versions = None
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -747,10 +747,18 @@
     return -1
 
 class LoopVersion(object):
+    _attrs_ = ('faildescrs', 'compile_data',
+               'operations', 'inputargs',
+               'renamed_inputargs')
+
+    faildescrs = []
+    compile_data = None
+    operations = None
+    inputargs = None
+    renamed_inputargs = None
 
     def __init__(self, loop):
         self.faildescrs = []
-        self._compiled = None
         if loop:
             self.operations = self.copy_operations(loop.operations) 
             idx = index_of_first(rop.LABEL, self.operations)
@@ -758,17 +766,13 @@
             label = self.operations[idx]
             self.inputargs = label.getarglist()
             self.renamed_inputargs = label.getarglist()
-        else:
-            self.operations = None
-            self.inputargs = None
-            self.renamed_inputargs = None
 
     def compiled(self):
         if self.operations is None:
             # root version must always be compiled
             return True
 
-        return self._compiled is not None
+        return self.compile_data is not None
 
     def copy_operations(self, operations):
         from rpython.jit.metainterp.compile import ResumeGuardDescr, CompileLoopVersionDescr 

From noreply at buildbot.pypy.org  Mon Aug 24 12:25:53 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 12:25:53 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: removed setting to none
Message-ID: <20150824102553.301851C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79190:5294b0296b79
Date: 2015-08-24 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/5294b0296b79/

Log:	removed setting to none

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -207,9 +207,7 @@
                                    faildescr, version.inputargs,
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
-            assert asminfo is not None
             version.compile_data = asminfo
-            faildescr.version = None
         # stitch the rest of the traces
         for lv in loop.versions:
             if not lv.compiled():
@@ -220,8 +218,6 @@
                 version = faildescr.version
                 if version and version.compiled():
                     cpu.stitch_bridge(faildescr, version.compile_data)
-                faildescr.version = None
-    loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,

From noreply at buildbot.pypy.org  Mon Aug 24 12:36:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 12:36:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: rework short preamble a bit
Message-ID: <20150824103613.490BA1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79191:260ee146ed8f
Date: 2015-08-24 11:13 +0200
http://bitbucket.org/pypy/pypy/changeset/260ee146ed8f/

Log:	rework short preamble a bit

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -346,8 +346,8 @@
     starting from short_boxes exported from the preamble. It will build
     the short preamble and necessary extra label arguments
     """
-    def __init__(self, short_boxes, short_inputargs, exported_infos,
-                 optimizer=None):
+    def __init__(self, label_args, short_boxes, short_inputargs,
+                 exported_infos, optimizer=None):
         for produced_op in short_boxes:
             op = produced_op.short_op.res
             preamble_op = produced_op.preamble_op
@@ -359,6 +359,9 @@
                     info = empty_info
             preamble_op.set_forwarded(info)
         self.short = []
+        self.label_dict = {}
+        for arg in label_args:
+            self.label_dict[arg] = None
         self.used_boxes = []
         self.short_preamble_jump = []
         self.extra_same_as = []
@@ -390,12 +393,17 @@
     def add_preamble_op(self, preamble_op):
         if preamble_op.invented_name:
             self.extra_same_as.append(preamble_op.op)
-        self.used_boxes.append(preamble_op.op)            
+        op = preamble_op.op
+        if op in self.label_dict:
+            return
+        self.label_dict[op] = None
+        self.used_boxes.append(op)
         self.short_preamble_jump.append(preamble_op.preamble_op)
 
-    def build_short_preamble(self, sb_jump):
+    def build_short_preamble(self):
         label_op = ResOperation(rop.LABEL, self.short_inputargs[:])
-        jump_op = ResOperation(rop.JUMP, sb_jump)
+        jump_op = ResOperation(rop.JUMP, self.short_preamble_jump)
+        # WARNING! the short_preamble_jump is shared on purpose
         TreeLoop.check_consistency_of(self.short_inputargs,
                                       self.short + [jump_op], check_descr=False)
         return [label_op] + self.short + [jump_op]
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -103,23 +103,28 @@
     def optimize_peeled_loop(self, start_label, end_jump, ops, state,
                              call_pure_results):
         self._check_no_forwarding([[start_label, end_jump], ops])
-        self.import_state(start_label, state)
-        self.potential_extra_ops = {}
         label_args = state.virtual_state.make_inputargs(
             start_label.getarglist(), self.optimizer)
+        self.import_state(start_label, state, label_args)
+        self.potential_extra_ops = {}
         self.optimizer.init_inparg_dict_from(label_args)
         info, _ = self.optimizer.propagate_all_forward(
             start_label.getarglist()[:], ops, call_pure_results, False)
         label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
-        target_token, extra = self.finalize_short_preamble(label_op,
+        target_token = self.finalize_short_preamble(label_op,
                                                     state.virtual_state)
         label_op.setdescr(target_token)
-        label_op.initarglist(label_op.getarglist() + extra)
+        extra = self.short_preamble_producer.used_boxes
+        label_args = [self.get_box_replacement(op) for op in label_args]
+        label_op.initarglist(label_args + extra)
         # force the boxes for virtual state to match
         x = state.virtual_state.make_inputargs(
             [self.get_box_replacement(x) for x in end_jump.getarglist()],
             self.optimizer, force_boxes=True)
         new_virtual_state = self.jump_to_existing_trace(end_jump)
+        # the short preamble could have grown, we reinitialize
+        # the label_op again
+        label_op.initarglist(label_args + extra)
         if new_virtual_state is not None:
             res = self.jump_to_preamble(start_label.getdescr(), end_jump,
                                          info)
@@ -169,18 +174,7 @@
     def finalize_short_preamble(self, label_op, virtual_state):
         sb = self.short_preamble_producer
         self.optimizer._clean_optimization_info(sb.short_inputargs)
-        d = {}
-        for arg in label_op.getarglist():
-            d[arg] = None
-        new_used_boxes = []
-        new_sb_jump = []
-        for i in range(len(sb.used_boxes)):
-            ub = sb.used_boxes[i]
-            if ub in d:
-                continue
-            new_used_boxes.append(ub)
-            new_sb_jump.append(sb.short_preamble_jump[i])
-        short_preamble = sb.build_short_preamble(new_sb_jump)
+        short_preamble = sb.build_short_preamble()
         jitcelltoken = label_op.getdescr()
         if jitcelltoken.target_tokens is None:
             jitcelltoken.target_tokens = []
@@ -190,7 +184,7 @@
         target_token.virtual_state = virtual_state
         target_token.short_preamble = short_preamble
         jitcelltoken.target_tokens.append(target_token)
-        return target_token, new_used_boxes
+        return target_token
 
     def jump_to_preamble(self, cell_token, jump_op, info):
         assert cell_token.target_tokens[0].virtual_state is None
@@ -238,6 +232,7 @@
 
     def inline_short_preamble(self, jump_args, short_inputargs, short_ops,
                               short_jump_op, patchguardop):
+        # warning!!!! short_jump_op might have arguments appended IN PLACE
         try:
             self._check_no_forwarding([short_inputargs, short_ops], False)
             assert len(short_inputargs) == len(jump_args)
@@ -285,7 +280,7 @@
                              short_boxes, renamed_inputargs,
                              short_inputargs)
 
-    def import_state(self, targetop, exported_state):
+    def import_state(self, targetop, exported_state, label_args):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
@@ -301,8 +296,9 @@
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
         self.short_preamble_producer = ShortPreambleBuilder(
-            exported_state.short_boxes, exported_state.short_inputargs,
-            exported_state.exported_infos, self.optimizer)
+            label_args, exported_state.short_boxes,
+            exported_state.short_inputargs, exported_state.exported_infos,
+            self.optimizer)
 
         for produced_op in exported_state.short_boxes:
             produced_op.produce_op(self, exported_state.exported_infos)

From noreply at buildbot.pypy.org  Mon Aug 24 12:36:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 12:36:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: a bit of work on preamble,
 thinking required now
Message-ID: <20150824103615.644461C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79192:a9e25b78318c
Date: 2015-08-24 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/a9e25b78318c/

Log:	a bit of work on preamble, thinking required now

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -103,9 +103,7 @@
     def optimize_peeled_loop(self, start_label, end_jump, ops, state,
                              call_pure_results):
         self._check_no_forwarding([[start_label, end_jump], ops])
-        label_args = state.virtual_state.make_inputargs(
-            start_label.getarglist(), self.optimizer)
-        self.import_state(start_label, state, label_args)
+        label_args = self.import_state(start_label, state)
         self.potential_extra_ops = {}
         self.optimizer.init_inparg_dict_from(label_args)
         info, _ = self.optimizer.propagate_all_forward(
@@ -280,7 +278,7 @@
                              short_boxes, renamed_inputargs,
                              short_inputargs)
 
-    def import_state(self, targetop, exported_state, label_args):
+    def import_state(self, targetop, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
@@ -295,6 +293,9 @@
                                             exported_state.exported_infos)
         # import the optimizer state, starting from boxes that can be produced
         # by short preamble
+        label_args = exported_state.virtual_state.make_inputargs(
+            targetop.getarglist(), self.optimizer)
+        
         self.short_preamble_producer = ShortPreambleBuilder(
             label_args, exported_state.short_boxes,
             exported_state.short_inputargs, exported_state.exported_infos,
@@ -303,6 +304,8 @@
         for produced_op in exported_state.short_boxes:
             produced_op.produce_op(self, exported_state.exported_infos)
 
+        return label_args
+
     def is_call_pure_with_exception(self, op):
         if op.is_call_pure():
             effectinfo = op.getdescr().get_extra_info()

From noreply at buildbot.pypy.org  Mon Aug 24 12:36:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 12:36:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: oops
Message-ID: <20150824103617.7FCF31C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79193:16f504e78a73
Date: 2015-08-24 12:36 +0200
http://bitbucket.org/pypy/pypy/changeset/16f504e78a73/

Log:	oops

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -113,7 +113,6 @@
                                                     state.virtual_state)
         label_op.setdescr(target_token)
         extra = self.short_preamble_producer.used_boxes
-        label_args = [self.get_box_replacement(op) for op in label_args]
         label_op.initarglist(label_args + extra)
         # force the boxes for virtual state to match
         x = state.virtual_state.make_inputargs(

From noreply at buildbot.pypy.org  Mon Aug 24 12:45:02 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 12:45:02 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: furhter stripping down the commit
Message-ID: <20150824104502.249361C0400@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79194:10602d3c86f4
Date: 2015-08-24 12:45 +0200
http://bitbucket.org/pypy/pypy/changeset/10602d3c86f4/

Log:	furhter stripping down the commit

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -194,30 +194,32 @@
     if loop.versions is not None:
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
-        root = loop.versions[0]
-        for faildescr in root.faildescrs:
-            version = faildescr.version
-            if not version or version.compiled():
-                continue
-            vl = create_empty_loop(metainterp)
-            vl.inputargs = version.inputargs
-            vl.operations = version.operations
-            vl.original_jitcell_token = jitcell_token
-            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
-                                   faildescr, version.inputargs,
-                                   version.operations, jitcell_token)
-            record_loop_or_bridge(metainterp_sd, vl)
-            version.compile_data = asminfo
+        for version in loop.versions:
+            if not version.compiled():
+                faildescr = version.faildescrs[0]
+                vl = create_empty_loop(metainterp)
+                vl.inputargs = version.inputargs
+                vl.operations = version.operations
+                vl.original_jitcell_token = jitcell_token
+                asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+                                       faildescr, version.inputargs,
+                                       version.operations, jitcell_token)
+                record_loop_or_bridge(metainterp_sd, vl)
+                version.compile_data = asminfo
+                version.operations = None
+                version.inputargs = None
         # stitch the rest of the traces
         for lv in loop.versions:
             if not lv.compiled():
                 # the version was never compiled, do not bother
                 # to assign it's fail descr
                 continue
-            for faildescr in lv.faildescrs:
-                version = faildescr.version
+            for faildescr in lv.faildescrs[1:]:
                 if version and version.compiled():
                     cpu.stitch_bridge(faildescr, version.compile_data)
+        for lv in loop.versions:
+            lv.compile_data = None
+            lv.faildescrs = []
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
@@ -525,7 +527,7 @@
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
                'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
-               'status', 'version')
+               'status')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -536,7 +538,6 @@
     rd_accum_list = None
 
     status = r_uint(0)
-    version = None
 
     def copy_all_attributes_from(self, other):
         assert isinstance(other, ResumeGuardDescr)
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -759,13 +759,12 @@
 
     def __init__(self, loop):
         self.faildescrs = []
-        if loop:
-            self.operations = self.copy_operations(loop.operations) 
-            idx = index_of_first(rop.LABEL, self.operations)
-            assert idx >= 0
-            label = self.operations[idx]
-            self.inputargs = label.getarglist()
-            self.renamed_inputargs = label.getarglist()
+        self.operations = self.copy_operations(loop.operations) 
+        idx = index_of_first(rop.LABEL, self.operations)
+        assert idx >= 0
+        label = self.operations[idx]
+        self.inputargs = label.getarglist()
+        self.renamed_inputargs = label.getarglist()
 
     def compiled(self):
         if self.operations is None:
@@ -804,7 +803,6 @@
         if not descr.loop_version():
             assert 0, "cannot register a guard that is not a CompileLoopVersionDescr"
         assert isinstance(descr, CompileLoopVersionDescr)
-        descr.version = version
         self.faildescrs.append(descr)
         # note: stitching a guard must resemble the order of the label
         # otherwise a wrong mapping is handed to the register allocator
@@ -844,6 +842,7 @@
     call_pure_results = None
     logops = None
     quasi_immutable_deps = None
+    versions = []
 
     def _token(*args):
         raise Exception("TreeLoop.token is killed")
@@ -856,7 +855,6 @@
 
     def __init__(self, name):
         self.name = name
-        self.versions = []
         # self.operations = list of ResOperations
         #   ops of the kind 'guard_xxx' contain a further list of operations,
         #   which may itself contain 'guard_xxx' and so on, making a tree.
@@ -911,15 +909,7 @@
         return None
 
     def snapshot(self):
-        if len(self.versions) == 0:
-            # create a root version, simplyfies the code in compile.py
-            self.versions.append(LoopVersion(None))
-        root_version = self.versions[0]
         version = LoopVersion(self)
-        if not we_are_translated():
-            print "LOOP SNAPSHOT"
-            for op in version.operations:
-                print "", op
         self.versions.append(version)
         return version
 

From noreply at buildbot.pypy.org  Mon Aug 24 13:30:43 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 13:30:43 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix one test
Message-ID: <20150824113043.55BFA1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79195:d0bea05ff042
Date: 2015-08-24 12:37 +0200
http://bitbucket.org/pypy/pypy/changeset/d0bea05ff042/

Log:	fix one test

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -55,7 +55,7 @@
         as a guiding heuristic whether making such guards makes
         sense. if None is passed in for op, no guard is ever generated, and
         this function degenerates to a generalization check."""
-        assert opinfo is None or isinstance(opinfo, info.AbstractInfo)
+        assert opinfo is None or opinfo.is_constant() or isinstance(opinfo, info.AbstractInfo)
         assert self.position != -1
         if self.position in state.renum:
             if state.renum[self.position] != other.position:

From noreply at buildbot.pypy.org  Mon Aug 24 13:30:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 13:30:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: maaaybe fix this test
Message-ID: <20150824113045.68B681C0400@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79196:74f0b0106b86
Date: 2015-08-24 13:24 +0200
http://bitbucket.org/pypy/pypy/changeset/74f0b0106b86/

Log:	maaaybe fix this test

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7231,7 +7231,8 @@
         p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
-        jump(p25, p187, i184)
+        p5 = same_as_r(p25)
+        jump(p25, p187, i184, p5)
         """
         short = """
         [p1, p187, i184]
@@ -7242,7 +7243,7 @@
         """
         expected = """
         [p25, p187, i184, p189]
-        jump(p25, p187, i184, p189)
+        jump(p189, p187, i184, p189) # <- XXX is this correct for bridges?
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -115,7 +115,7 @@
         extra = self.short_preamble_producer.used_boxes
         label_op.initarglist(label_args + extra)
         # force the boxes for virtual state to match
-        x = state.virtual_state.make_inputargs(
+        state.virtual_state.make_inputargs(
             [self.get_box_replacement(x) for x in end_jump.getarglist()],
             self.optimizer, force_boxes=True)
         new_virtual_state = self.jump_to_existing_trace(end_jump)
@@ -219,7 +219,7 @@
             args = target_virtual_state.make_inputargs(args,
                 self.optimizer)
             extra = self.inline_short_preamble(pass_to_short,
-                short_preamble[0].getarglist(), short_preamble[1:-1],
+                short_preamble[0].getarglist(), short_preamble,
                 short_preamble[-1].getarglist(), self.optimizer.patchguardop)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
@@ -236,16 +236,19 @@
             for i in range(len(jump_args)):
                 short_inputargs[i].set_forwarded(None)
                 self.make_equal_to(short_inputargs[i], jump_args[i])
-            for op in short_ops:
+            i = 1
+            while i < len(short_ops) - 1:
+                op = short_ops[i]
                 if op.is_guard():
                     op = self.replace_op_with(op, op.getopnum(),
                                     descr=compile.ResumeAtPositionDescr())
                     op.rd_snapshot = patchguardop.rd_snapshot
                     op.rd_frame_info_list = patchguardop.rd_frame_info_list
                 self.optimizer.send_extra_operation(op)
-            res = [self.optimizer.get_box_replacement(op) for op in
-                    short_jump_op]
-            return res
+                i += 1
+            for arg in jump_args:
+                self.optimizer.force_box(self.get_box_replacement(arg))
+            return [self.get_box_replacement(box) for box in short_jump_op]
         finally:
             for op in short_inputargs:
                 op.set_forwarded(None)

From noreply at buildbot.pypy.org  Mon Aug 24 13:30:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 13:30:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: be a bit more careful
Message-ID: <20150824113047.9BC5E1C1155@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79197:bb871fddbdb0
Date: 2015-08-24 13:30 +0200
http://bitbucket.org/pypy/pypy/changeset/bb871fddbdb0/

Log:	be a bit more careful

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -218,7 +218,7 @@
                 self.optimizer, append_virtuals=True)
             args = target_virtual_state.make_inputargs(args,
                 self.optimizer)
-            extra = self.inline_short_preamble(pass_to_short,
+            extra = self.inline_short_preamble(pass_to_short, args,
                 short_preamble[0].getarglist(), short_preamble,
                 short_preamble[-1].getarglist(), self.optimizer.patchguardop)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
@@ -227,7 +227,8 @@
             return None # explicit because the return can be non-None
         return virtual_state
 
-    def inline_short_preamble(self, jump_args, short_inputargs, short_ops,
+    def inline_short_preamble(self, jump_args, args_no_virtuals,
+                              short_inputargs, short_ops,
                               short_jump_op, patchguardop):
         # warning!!!! short_jump_op might have arguments appended IN PLACE
         try:
@@ -246,7 +247,8 @@
                     op.rd_frame_info_list = patchguardop.rd_frame_info_list
                 self.optimizer.send_extra_operation(op)
                 i += 1
-            for arg in jump_args:
+            # force all of them except the virtuals
+            for arg in args_no_virtuals + short_jump_op:
                 self.optimizer.force_box(self.get_box_replacement(arg))
             return [self.get_box_replacement(box) for box in short_jump_op]
         finally:

From noreply at buildbot.pypy.org  Mon Aug 24 13:33:50 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 24 Aug 2015 13:33:50 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: rebreak this test to be
	discussed
Message-ID: <20150824113350.8F8631C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79198:fa0b3e053262
Date: 2015-08-24 13:33 +0200
http://bitbucket.org/pypy/pypy/changeset/fa0b3e053262/

Log:	rebreak this test to be discussed

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7243,7 +7243,7 @@
         """
         expected = """
         [p25, p187, i184, p189]
-        jump(p189, p187, i184, p189) # <- XXX is this correct for bridges?
+        jump(p25, p187, i184, p189) # <- XXX is this correct for bridges?
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 

From noreply at buildbot.pypy.org  Mon Aug 24 13:41:42 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 13:41:42 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: memory error does not occur anymore
Message-ID: <20150824114142.12D2E1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79199:fbcc105f757d
Date: 2015-08-24 13:41 +0200
http://bitbucket.org/pypy/pypy/changeset/fbcc105f757d/

Log:	memory error does not occur anymore

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -592,9 +592,8 @@
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
 
     def stitch_bridge(self, faildescr, target):
-        pass
-        #assert target != 0
-        #self.patch_jump_for_descr(faildescr, target.rawstart)
+        assert target.rawstart != 0
+        self.patch_jump_for_descr(faildescr, target.rawstart)
 
     def write_pending_failure_recoveries(self, regalloc):
         # for each pending guard, generate the code of the recovery stub
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -162,7 +162,7 @@
 
     if loop.versions is not None:
         # every different loop version must update their target tokens
-        for version in loop.versions[1:]:
+        for version in loop.versions:
             version.update_token(jitcell_token, all_target_tokens)
 
     if not loop.quasi_immutable_deps:
@@ -195,7 +195,7 @@
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
         for version in loop.versions:
-            if not version.compiled():
+            if not version.compiled() and version.faildescrs:
                 faildescr = version.faildescrs[0]
                 vl = create_empty_loop(metainterp)
                 vl.inputargs = version.inputargs
@@ -214,9 +214,11 @@
                 # the version was never compiled, do not bother
                 # to assign it's fail descr
                 continue
-            for faildescr in lv.faildescrs[1:]:
-                if version and version.compiled():
-                    cpu.stitch_bridge(faildescr, version.compile_data)
+            for i,faildescr in enumerate(lv.faildescrs):
+                if i == 0:
+                    continue
+                assert isinstance(faildescr, AbstractFailDescr)
+                cpu.stitch_bridge(faildescr, lv.compile_data)
         for lv in loop.versions:
             lv.compile_data = None
             lv.faildescrs = []
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -757,9 +757,11 @@
     inputargs = None
     renamed_inputargs = None
 
-    def __init__(self, loop):
+    def __init__(self, operations):
         self.faildescrs = []
-        self.operations = self.copy_operations(loop.operations) 
+        self.operations = operations
+
+    def setup_once(self):
         idx = index_of_first(rop.LABEL, self.operations)
         assert idx >= 0
         label = self.operations[idx]
@@ -773,36 +775,9 @@
 
         return self.compile_data is not None
 
-    def copy_operations(self, operations):
-        from rpython.jit.metainterp.compile import ResumeGuardDescr, CompileLoopVersionDescr 
-        ignore = (rop.DEBUG_MERGE_POINT,)
-        oplist = []
-        for op in operations:
-            if op.getopnum() in ignore:
-                continue
-            cloned = op.clone()
-            oplist.append(cloned)
-            if cloned.is_guard():
-                olddescr = cloned.getdescr()
-                if not olddescr:
-                    continue
-                descr = olddescr.clone()
-                cloned.setdescr(descr)
-                if olddescr.loop_version():
-                    # copy the version
-                    assert isinstance(descr, CompileLoopVersionDescr)
-                    assert isinstance(olddescr, CompileLoopVersionDescr)
-                    descr.version = olddescr.version
-                    self.faildescrs.append(descr)
-        return oplist
-
     def register_guard(self, op, version):
-        from rpython.jit.metainterp.compile import CompileLoopVersionDescr
         assert isinstance(op, GuardResOp)
         descr = op.getdescr()
-        if not descr.loop_version():
-            assert 0, "cannot register a guard that is not a CompileLoopVersionDescr"
-        assert isinstance(descr, CompileLoopVersionDescr)
         self.faildescrs.append(descr)
         # note: stitching a guard must resemble the order of the label
         # otherwise a wrong mapping is handed to the register allocator
@@ -909,10 +884,29 @@
         return None
 
     def snapshot(self):
-        version = LoopVersion(self)
+        oplist = self.copy_operations(self.operations)
+        version = LoopVersion(oplist)
+        version.setup_once()
         self.versions.append(version)
         return version
 
+    def copy_operations(self, operations):
+        ignore = (rop.DEBUG_MERGE_POINT,)
+        oplist = []
+        for op in operations:
+            if op.getopnum() in ignore:
+                continue
+            cloned = op.clone()
+            oplist.append(cloned)
+            if cloned.is_guard():
+                olddescr = cloned.getdescr()
+                if not olddescr:
+                    continue
+                descr = olddescr.clone()
+                cloned.setdescr(descr)
+        return oplist
+
+
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
 
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -152,10 +152,10 @@
         assert operations[self.index] is self.op
         operations[self.index] = None
         descr = self.op.getdescr()
-        if descr and descr.loop_version():
-            from rpython.jit.metainterp.compile import CompileLoopVersionDescr
-            assert isinstance(descr, CompileLoopVersionDescr)
-            descr.version = None
+        #if descr and descr.loop_version():
+        #    from rpython.jit.metainterp.compile import CompileLoopVersionDescr
+        #    assert isinstance(descr, CompileLoopVersionDescr)
+        #    descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None
 
@@ -266,20 +266,18 @@
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
         #
-        assert len(loop.versions) == 2, "need one orignal loop version (before trans) and the current one"
-        root_version = loop.versions[0]
-        version = loop.versions[1]
-
+        assert len(loop.versions) == 1
+        version = loop.versions[0]
         for op in loop.operations:
             if not op.is_guard():
                 continue
             descr = op.getdescr()
             if descr.loop_version():
-                root_version.register_guard(op, version)
+                version.register_guard(op, version)
 
-        if user_code:
-            version = loop.snapshot()
-            self.eliminate_array_bound_checks(loop, root_version, version)
+        #if user_code:
+        #    version = loop.snapshot()
+        #    self.eliminate_array_bound_checks(loop, root_version, version)
 
     def emit_operation(self, op):
         self.renamer.rename(op)

From noreply at buildbot.pypy.org  Mon Aug 24 13:51:41 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 13:51:41 +0200 (CEST)
Subject: [pypy-commit] pypy memoryerror3: still working
Message-ID: <20150824115141.5CFE31C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: memoryerror3
Changeset: r79200:f4a6026863c1
Date: 2015-08-24 13:51 +0200
http://bitbucket.org/pypy/pypy/changeset/f4a6026863c1/

Log:	still working

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -759,15 +759,33 @@
 
     def __init__(self, operations):
         self.faildescrs = []
-        self.operations = operations
-
-    def setup_once(self):
+        self.operations = self.copy_operations(operations)
         idx = index_of_first(rop.LABEL, self.operations)
         assert idx >= 0
         label = self.operations[idx]
         self.inputargs = label.getarglist()
         self.renamed_inputargs = label.getarglist()
 
+    def setup_once(self):
+        pass
+
+    def copy_operations(self, operations):
+        ignore = (rop.DEBUG_MERGE_POINT,)
+        oplist = []
+        for op in operations:
+            if op.getopnum() in ignore:
+                continue
+            cloned = op.clone()
+            oplist.append(cloned)
+            if cloned.is_guard():
+                olddescr = cloned.getdescr()
+                if not olddescr:
+                    continue
+                descr = olddescr.clone()
+                cloned.setdescr(descr)
+        return oplist
+
+
     def compiled(self):
         if self.operations is None:
             # root version must always be compiled
@@ -884,8 +902,8 @@
         return None
 
     def snapshot(self):
-        oplist = self.copy_operations(self.operations)
-        version = LoopVersion(oplist)
+        #oplist = self.copy_operations(self.operations)
+        version = LoopVersion(self.operations)
         version.setup_once()
         self.versions.append(version)
         return version

From noreply at buildbot.pypy.org  Mon Aug 24 15:26:13 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Aug 2015 15:26:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: backout 40b6f6640e9e,
 which was a backout of d28b92c9f186
Message-ID: <20150824132613.457B41C1192@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79201:a206f5fe2639
Date: 2015-08-24 14:47 +0200
http://bitbucket.org/pypy/pypy/changeset/a206f5fe2639/

Log:	backout 40b6f6640e9e, which was a backout of d28b92c9f186

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -101,9 +101,6 @@
     def is_object(self):
         return self._is_object
 
-    def as_vtable_size_descr(self):
-        return self
-
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
             self._corresponding_vtable))
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -59,9 +59,6 @@
         return False
 
 class SizeDescrWithVTable(SizeDescr):
-    def as_vtable_size_descr(self):
-        return self
-
     def is_object(self):
         return True
 
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -121,7 +121,8 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- GETFIELD_GC ----------
-            if op.getopnum() == rop.GETFIELD_GC:
+            if op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F,
+                                 rop.GETFIELD_GC_R):
                 self.handle_getfield_gc(op)
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
@@ -194,7 +195,7 @@
         followed by a bunch of 'setfields', and the 'pending_zeros'
         optimization we do here is meant for this case."""
         self.emit_pending_zeros()
-        self.newops.append(op)
+        self.emit_op(op)
 
     # ----------
 
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -8,7 +8,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, lltype, llmemory
 from rpython.rtyper import rclass
@@ -52,7 +51,6 @@
         O = lltype.GcStruct('O', ('parent', rclass.OBJECT),
                                  ('x', lltype.Signed))
         o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -7,7 +7,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -64,7 +63,6 @@
                                  ('x', lltype.Signed))
         o_descr = self.cpu.sizeof(O, True)
         o_vtable = globals()['o_vtable']
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1950,8 +1950,7 @@
         cpu = self.cpu
         t_box, T_box, descr = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
-        heaptracker.register_known_gctype(cpu, vtable, self.T)
+            llmemory.cast_int_to_adr(T_box.getint()), rclass.CLASSTYPE)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         assert r1 != r2
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -123,8 +123,6 @@
         vtable.name = rclass.alloc_array_name(name)
         self.structure_types_and_vtables.append((S, vtable))
         #
-        heaptracker.register_known_gctype(self.cpu, vtable, S)
-        #
         return S, vtable
 
     def get_random_structure(self, r, has_vtable=False):
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -74,7 +74,6 @@
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
         self.assembler.finished(self.callcontrol.callinfocollection)
-        heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -88,57 +88,6 @@
 
 # ____________________________________________________________
 
-VTABLETYPE = rclass.CLASSTYPE
-
-def register_known_gctype(cpu, vtable, STRUCT):
-    # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
-    sizedescr = cpu.sizeof(STRUCT, vtable)
-    assert sizedescr.as_vtable_size_descr() is sizedescr
-    if getattr(sizedescr, '_corresponding_vtable', None):
-        assert sizedescr._corresponding_vtable == vtable
-        return
-    assert lltype.typeOf(vtable) == VTABLETYPE
-    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-        cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._vtable_to_descr_dict = None
-    cpu.tracker._all_size_descrs_with_vtable.append(sizedescr)
-    sizedescr._corresponding_vtable = vtable
-
-def finish_registering(cpu):
-    # annotation hack for small examples which have no vtable at all
-    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
-    pass
-
-def vtable2descr(cpu, vtable):
-    assert lltype.typeOf(vtable) is lltype.Signed
-    vtable = int2adr(vtable)
-    if we_are_translated():
-        # Build the dict {vtable: sizedescr} at runtime.
-        # This is necessary because the 'vtables' are just pointers to
-        # static data, so they can't be used as keys in prebuilt dicts.
-        d = cpu.tracker._vtable_to_descr_dict
-        if d is None:
-            d = cpu.tracker._vtable_to_descr_dict = {}
-            for descr in cpu.tracker._all_size_descrs_with_vtable:
-                key = descr._corresponding_vtable
-                key = llmemory.cast_ptr_to_adr(key)
-                d[key] = descr
-        return d[vtable]
-    else:
-        vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE)
-        for descr in cpu.tracker._all_size_descrs_with_vtable:
-            if descr._corresponding_vtable == vtable:
-                return descr
-        raise KeyError(vtable)
-
-def descr2vtable(cpu, descr):
-    from rpython.jit.metainterp import history
-    assert isinstance(descr, history.AbstractDescr)
-    vtable = descr.as_vtable_size_descr()._corresponding_vtable
-    vtable = llmemory.cast_ptr_to_adr(vtable)
-    return adr2int(vtable)
 
 def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None,
                     get_field_descr=None):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -932,7 +932,6 @@
                     return self._do_builtin_call(op, 'alloc_with_del', [],
                                                  extra=(RESULT, vtable),
                                                  extrakey=STRUCT)
-            heaptracker.register_known_gctype(self.cpu, vtable, STRUCT)
             opname = 'new_with_vtable'
         else:
             opname = 'new'
@@ -1936,10 +1935,6 @@
     # VirtualRefs.
 
     def _handle_virtual_ref_call(self, op, oopspec_name, args):
-        vrefinfo = self.callcontrol.virtualref_info
-        heaptracker.register_known_gctype(self.cpu,
-                                          vrefinfo.jit_virtual_ref_vtable,
-                                          vrefinfo.JIT_VIRTUAL_REF)
         return SpaceOperation(oopspec_name, list(args), op.result)
 
     # -----------
diff --git a/rpython/jit/codewriter/test/test_codewriter.py b/rpython/jit/codewriter/test/test_codewriter.py
--- a/rpython/jit/codewriter/test/test_codewriter.py
+++ b/rpython/jit/codewriter/test/test_codewriter.py
@@ -25,8 +25,6 @@
 class FakeSizeDescr(AbstractDescr):
     def __init__(self, STRUCT):
         self.STRUCT = STRUCT
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeArrayDescr(AbstractDescr):
     def __init__(self, ARRAY):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -39,8 +39,6 @@
         self.oopspecindex = oopspecindex
     def __repr__(self):
         return ''
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeDict(object):
     def __getitem__(self, key):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -50,8 +50,7 @@
         return FakeDescr(('sizedescr', STRUCT))
 
 class FakeDescr(tuple):
-    def as_vtable_size_descr(self):
-        return self
+    pass
 
 class FakeLink:
     args = []
@@ -569,9 +568,6 @@
     op1 = Transformer(cpu).rewrite_operation(op)
     assert op1.opname == 'new_with_vtable'
     assert op1.args == [('sizedescr', S)]
-    #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check]
-    vtable_int = heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtable))
-    assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0]
 
 def test_malloc_new_with_destructor():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -242,6 +242,11 @@
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
 
+    def _get_buffer(self):
+        buffer = self.buffer
+        assert buffer is not None
+        return buffer
+
     def getitem_raw(self, offset, itemsize, descr):
         if not self.is_virtual():
             raise InvalidRawOperation
@@ -249,37 +254,39 @@
             # for the test above: it's not enough to check is_virtual()
             # on the original object, because it might be a VRawSliceValue
             # instead.  If it is a virtual one, then we'll reach here anway.
-        return self.buffer.read_value(offset, itemsize, descr)
+        return self._get_buffer().read_value(offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         if not self.is_virtual():
             raise InvalidRawOperation
-        self.buffer.write_value(offset, itemsize, descr, itemop)
+        self._get_buffer().write_value(offset, itemsize, descr, itemop)
 
     def is_virtual(self):
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
         self.size = -1
-        for i in range(len(self.buffer.offsets)):
+        buffer = self._get_buffer()
+        for i in range(len(buffer.offsets)):
             # write the value
-            offset = self.buffer.offsets[i]
-            descr = self.buffer.descrs[i]
-            itembox = self.buffer.values[i]
+            offset = buffer.offsets[i]
+            descr = buffer.descrs[i]
+            itembox = buffer.values[i]
             setfield_op = ResOperation(rop.RAW_STORE,
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(setfield_op)
 
     def _visitor_walk_recursive(self, op, visitor, optimizer):
-        itemboxes = self.buffer.values
+        itemboxes = self._get_buffer().values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
+        buffer = self._get_buffer()
         return visitor.visit_vrawbuffer(self.size,
-                                        self.buffer.offsets[:],
-                                        self.buffer.descrs[:])
+                                        buffer.offsets[:],
+                                        buffer.descrs[:])
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -12,7 +12,6 @@
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.logger import LogOperations
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import OpParser
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
@@ -352,17 +351,6 @@
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
     vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
 
-    register_known_gctype(cpu, node_vtable,  NODE)
-    register_known_gctype(cpu, node_vtable2, NODE2)
-    register_known_gctype(cpu, node_vtable3, NODE3)
-    register_known_gctype(cpu, u_vtable,     U)
-    register_known_gctype(cpu, simple_vtable,     SIMPLE)
-    register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
-    register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
-    register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
-    register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
-    register_known_gctype(cpu, w_root_vtable, W_ROOT)
-
     namespace = locals()
 
 # ____________________________________________________________
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,5 +1,4 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.codewriter.heaptracker import descr2vtable
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
@@ -199,7 +198,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr()))
+        known_class = ConstInt(op.getdescr().get_vtable())
         self.make_virtual(known_class, op, op.getdescr())
 
     def optimize_NEW(self, op):
diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -597,8 +597,7 @@
                 self._no = no
 
         class FakeDescr:
-            def as_vtable_size_descr(self):
-                return self
+            pass
 
         class FakeCPU(object):
             supports_floats = False

From noreply at buildbot.pypy.org  Mon Aug 24 15:26:15 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Aug 2015 15:26:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix, in-progress
Message-ID: <20150824132615.69B6F1C1192@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79202:8851cf14646f
Date: 2015-08-24 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/8851cf14646f/

Log:	fix, in-progress

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -90,7 +90,9 @@
 
 class SizeDescr(AbstractDescr):
     def __init__(self, S, vtable, runner):
+        assert not isinstance(vtable, bool)
         self.S = S
+        self._vtable = vtable
         self._is_object = vtable is not None
         self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
                                     get_field_descr=LLGraphCPU.fielddescrof)
@@ -102,8 +104,7 @@
         return self._is_object
 
     def get_vtable(self):
-        return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
-            self._corresponding_vtable))
+        return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self._vtable))
 
     def count_fields_if_immutable(self):
         return heaptracker.count_fields_if_immutable(self.S)
@@ -692,7 +693,7 @@
         result = lltype.malloc(descr.S, zero=True)
         result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result)
         result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE,
-                                                descr._corresponding_vtable)
+                                                descr._vtable)
         return lltype.cast_opaque_ptr(llmemory.GCREF, result)
 
     def bh_new_array(self, length, arraydescr):
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -8,10 +8,6 @@
     total_freed_loops = 0
     total_freed_bridges = 0
 
-    # for heaptracker
-    # _all_size_descrs_with_vtable = None
-    _vtable_to_descr_dict = None
-
 class AbstractCPU(object):
     supports_floats = False
     supports_longlong = False
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1820,7 +1820,6 @@
                 p = p.parent
         else:
             descr = cpu.sizeof(T, vtable_for_T)
-            descr._corresponding_vtable = vtable_for_T
         t = lltype.malloc(T)
         if T == self.T:
             t.parent.parent.typeptr = vtable_for_T
diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py
--- a/rpython/jit/backend/x86/test/test_regalloc2.py
+++ b/rpython/jit/backend/x86/test/test_regalloc2.py
@@ -335,11 +335,7 @@
     cls1.subclassrange_min = 1
     cls1.subclassrange_max = 3
     if S is not None:
-        descr = cpu.sizeof(S)
-        if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-            cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._all_size_descrs_with_vtable.append(descr)
-        descr._corresponding_vtable = cls1
+        descr = cpu.sizeof(S, cls1)    # xxx may be not needed any more
     return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(cls1), "symbolic")
 
 def test_bug_2():
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -89,8 +89,6 @@
     metainterp.history = History()
     metainterp.history.operations = loop.operations[:-1]
     metainterp.history.inputargs = loop.inputargs[:]
-    cpu.tracker._all_size_descrs_with_vtable = (
-        LLtypeMixin.cpu.tracker._all_size_descrs_with_vtable)
     #
     greenkey = 'faked'
     target_token = compile_loop(metainterp, greenkey, 0,

From noreply at buildbot.pypy.org  Mon Aug 24 15:26:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Aug 2015 15:26:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: Fix fix fix
Message-ID: <20150824132617.851D31C1192@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79203:c04209abbad1
Date: 2015-08-24 15:26 +0200
http://bitbucket.org/pypy/pypy/changeset/c04209abbad1/

Log:	Fix fix fix

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -96,6 +96,7 @@
         self._is_object = vtable is not None
         self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
                                     get_field_descr=LLGraphCPU.fielddescrof)
+        self._runner = runner
 
     def get_all_fielddescrs(self):
         return self.all_fielddescrs
@@ -104,6 +105,10 @@
         return self._is_object
 
     def get_vtable(self):
+        assert self._vtable is not None
+        if self._vtable is Ellipsis:
+            self._vtable = heaptracker.get_vtable_for_gcstruct(self._runner,
+                                                               self.S)
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self._vtable))
 
     def count_fields_if_immutable(self):
@@ -411,11 +416,14 @@
     def sizeof(self, S, vtable):
         key = ('size', S)
         try:
-            return self.descrs[key]
+            descr = self.descrs[key]
         except KeyError:
             descr = SizeDescr(S, vtable, self)
             self.descrs[key] = descr
-            return descr
+        if descr._is_object and vtable is not Ellipsis:
+            assert vtable
+            heaptracker.testing_gcstruct2vtable.setdefault(S, vtable)
+        return descr
 
     def fielddescrof(self, S, fieldname):
         key = ('field', S, fieldname)
@@ -424,8 +432,7 @@
         except KeyError:
             descr = FieldDescr(S, fieldname)
             self.descrs[key] = descr
-            is_obj = heaptracker.has_gcstruct_a_vtable(S)
-            descr.parent_descr = self.sizeof(S, is_obj)
+            descr.parent_descr = self.sizeof(S, Ellipsis)
             if self.vinfo_for_tests is not None:
                 descr.vinfo = self.vinfo_for_tests
             return descr
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -195,10 +195,10 @@
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         if STRUCT is rclass.OBJECT:
-            fielddescr.parent_descr = get_size_descr(gccache, STRUCT, None)
+            vtable = None
         else:
-            fielddescr.parent_descr = get_size_descr(gccache, STRUCT,
-                            heaptracker.get_vtable_for_gcstruct(gccache, STRUCT))
+            vtable = heaptracker.get_vtable_for_gcstruct(gccache, STRUCT)
+        fielddescr.parent_descr = get_size_descr(gccache, STRUCT, vtable)
         return fielddescr
 
 def get_type_flag(TYPE):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -67,12 +67,14 @@
     if not has_gcstruct_a_vtable(GCSTRUCT):
         return None
     setup_cache_gcstruct2vtable(gccache)
-    return gccache._cache_gcstruct2vtable[GCSTRUCT]
+    try:
+        return gccache._cache_gcstruct2vtable[GCSTRUCT]
+    except KeyError:
+        return testing_gcstruct2vtable[GCSTRUCT]
 
 def setup_cache_gcstruct2vtable(gccache):
     if not hasattr(gccache, '_cache_gcstruct2vtable'):
         cache = {}
-        cache.update(testing_gcstruct2vtable)
         if gccache.rtyper:
             for rinstance in gccache.rtyper.instance_reprs.values():
                 cache[rinstance.lowleveltype.TO] = rinstance.rclass.getvtable()
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -121,9 +121,9 @@
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
     #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
-    nodesize = cpu.sizeof(NODE, True)
-    nodesize2 = cpu.sizeof(NODE2, True)
-    nodesize3 = cpu.sizeof(NODE3, True)
+    nodesize = cpu.sizeof(NODE, node_vtable)
+    nodesize2 = cpu.sizeof(NODE2, node_vtable2)
+    nodesize3 = cpu.sizeof(NODE3, node_vtable3)
     valuedescr = cpu.fielddescrof(NODE, 'value')
     floatdescr = cpu.fielddescrof(NODE, 'floatval')
     chardescr = cpu.fielddescrof(NODE, 'charval')
@@ -139,7 +139,7 @@
     QUASI = lltype.GcStruct('QUASIIMMUT', ('inst_field', lltype.Signed),
                             ('mutate_field', rclass.OBJECTPTR),
                             hints={'immutable_fields': accessor})
-    quasisize = cpu.sizeof(QUASI, False)
+    quasisize = cpu.sizeof(QUASI, None)
     quasi = lltype.malloc(QUASI, immortal=True)
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
@@ -162,8 +162,8 @@
     intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval')
     immut_intval = cpu.fielddescrof(INTOBJ_IMMUT, 'intval')
-    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, True)
-    immut_descr = cpu.sizeof(INTOBJ_IMMUT, True)
+    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, intobj_noimmut_vtable)
+    immut_descr = cpu.sizeof(INTOBJ_IMMUT, intobj_immut_vtable)
 
     PTROBJ_IMMUT = lltype.GcStruct('PTROBJ_IMMUT', ('parent', OBJECT),
                                             ('ptrval', lltype.Ptr(OBJECT)),
@@ -178,7 +178,7 @@
     # a GcStruct not inheriting from OBJECT
     S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed),
                         ('b', lltype.Ptr(NODE)))
-    ssize = cpu.sizeof(S, False)
+    ssize = cpu.sizeof(S, None)
     adescr = cpu.fielddescrof(S, 'a')
     abisdescr = cpu.fielddescrof(S, 'abis')
     bdescr = cpu.fielddescrof(S, 'b')
@@ -201,7 +201,7 @@
     inst_w_list = cpu.fielddescrof(W_ROOT, 'inst_w_list')
     w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     
-    tsize = cpu.sizeof(T, False)
+    tsize = cpu.sizeof(T, None)
     cdescr = cpu.fielddescrof(T, 'c')
     ddescr = cpu.fielddescrof(T, 'd')
     arraydescr3 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(NODE)))
@@ -214,10 +214,10 @@
     SIMPLE = lltype.GcStruct('simple',
         ('parent', OBJECT),
         ('value', lltype.Signed))
-    simpledescr = cpu.sizeof(SIMPLE, True)
     simplevalue = cpu.fielddescrof(SIMPLE, 'value')
     simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    usize = cpu.sizeof(U, True)
+    simpledescr = cpu.sizeof(SIMPLE, simple_vtable)
+    #usize = cpu.sizeof(U, ...)
     onedescr = cpu.fielddescrof(U, 'one')
 
     FUNC = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -349,7 +349,7 @@
 
     jit_virtual_ref_vtable = vrefinfo.jit_virtual_ref_vtable
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
-    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
+    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, jit_virtual_ref_vtable)
 
     namespace = locals()
 

From noreply at buildbot.pypy.org  Mon Aug 24 15:27:14 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 15:27:14 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: memory error does not happen
 anymore on vecopt-merge
Message-ID: <20150824132714.470281C1192@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79204:9e571b42712d
Date: 2015-08-24 15:27 +0200
http://bitbucket.org/pypy/pypy/changeset/9e571b42712d/

Log:	memory error does not happen anymore on vecopt-merge

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -591,14 +591,14 @@
                                                        rawstart, fullsize)
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos, rawstart)
 
-    def stitch_bridge(self, faildescr, version):
+    def stitch_bridge(self, faildescr, target):
         """ Stitching means that one can enter a bridge with a complete different register
             allocation. This needs remapping which is done here for both normal registers
             and accumulation registers.
             Why? Because this only generates a very small junk of memory, instead of
             duplicating the loop assembler for each faildescr!
         """
-        asminfo, bridge_faildescr, compiled_version, looptoken = version._compiled
+        asminfo, bridge_faildescr, version, looptoken = target
         assert isinstance(bridge_faildescr, ResumeGuardDescr)
         assert isinstance(faildescr, ResumeGuardDescr)
         assert asminfo.rawstart != 0
@@ -612,7 +612,7 @@
         self.mc.force_frame_size(DEFAULT_FRAME_BYTES)
         # if accumulation is saved at the guard, we need to update it here!
         guard_locs = self.rebuild_faillocs_from_descr(faildescr, version.inputargs)
-        bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, compiled_version.inputargs)
+        bridge_locs = self.rebuild_faillocs_from_descr(bridge_faildescr, version.inputargs)
         guard_accum_info = faildescr.rd_accum_list
         # O(n^2), but usually you only have at most 1 fail argument
         while guard_accum_info:
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -162,7 +162,7 @@
 
     if loop.versions is not None:
         # every different loop version must update their target tokens
-        for version in loop.versions[1:]:
+        for version in loop.versions:
             version.update_token(jitcell_token, all_target_tokens)
 
     if not loop.quasi_immutable_deps:
@@ -194,12 +194,12 @@
     if loop.versions:
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
-        root = loop.versions[0]
-        for faildescr in root.faildescrs:
-            assert isinstance(faildescr, CompileLoopVersionDescr)
-            version = faildescr.version
-            if not version or version.compiled():
+        to_stitch = []
+        for version in loop.versions:
+            if not version.faildescrs:
                 continue
+            faildescr = version.faildescrs[0]
+            assert isinstance(faildescr, ResumeGuardDescr)
             vl = create_empty_loop(metainterp)
             vl.inputargs = version.inputargs
             vl.operations = version.operations
@@ -209,20 +209,14 @@
                                    version.operations, jitcell_token)
             record_loop_or_bridge(metainterp_sd, vl)
             assert asminfo is not None
-            version._compiled = (asminfo, faildescr, faildescr.version, jitcell_token)
-            faildescr.version = None
+
+            for i,fd in enumerate(version.faildescrs):
+                if i == 0:
+                    continue
+                to_stitch.append((fd, (asminfo, faildescr, version, jitcell_token)))
         # stitch to the trace loop
-        for lv in loop.versions:
-            if not lv.compiled():
-                # the version was never compiled, do not bother
-                # to assign it's fail descr
-                continue
-            for faildescr in lv.faildescrs:
-                assert isinstance(faildescr, CompileLoopVersionDescr)
-                version = faildescr.version
-                if version and version.compiled():
-                    cpu.stitch_bridge(faildescr, version)
-                faildescr.version = None
+        for fd, param in to_stitch:
+            cpu.stitch_bridge(fd, param)
     loop.versions = None
 
 def compile_retrace(metainterp, greenkey, start,
@@ -531,7 +525,7 @@
 class ResumeGuardDescr(ResumeDescr):
     _attrs_ = ('rd_numb', 'rd_count', 'rd_consts', 'rd_virtuals',
                'rd_frame_info_list', 'rd_pendingfields', 'rd_accum_list',
-               'status')
+               'status', 'version')
     
     rd_numb = lltype.nullptr(NUMBERING)
     rd_count = 0
@@ -542,6 +536,7 @@
     rd_accum_list = None
 
     status = r_uint(0)
+    version = None
 
     def copy_all_attributes_from(self, other):
         assert isinstance(other, ResumeGuardDescr)
@@ -746,11 +741,6 @@
 class CompileLoopVersionDescr(ResumeGuardDescr):
     guard_opnum = rop.GUARD_EARLY_EXIT
 
-    operations = None
-    inputargs = None
-    faillocs = None
-    version = None
-
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         assert 0, "this guard must never fail"
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -744,60 +744,31 @@
         create one instance and attach it to a guard descr.
         If not attached to a descriptor, it will not be compiled.
     """
-    def __init__(self, loop):
-        self.faildescrs = []
-        self._compiled = (None,None,None,None)
-        if loop:
-            self.operations = self.copy_operations(loop.operations) 
+    _compiled = (None,None,None,None)
+    inputargs = None
+    renamed_inputargs = None
+
+    def __init__(self, operations):
+        self.faildescrs = None
+        self.stitchdescr = {}
+        self.operations = operations
+
+    def setup_once(self):
+        if self.operations:
             idx = index_of_first(rop.LABEL, self.operations)
             assert idx >= 0
             label = self.operations[idx]
             self.inputargs = label.getarglist()
             self.renamed_inputargs = label.getarglist()
-        else:
-            self.operations = None
-            self.inputargs = None
-            self.renamed_inputargs = None
+            # register the faildescr for later stitching
+            for op in self.operations:
+                if op.is_guard():
+                    descr = op.getdescr()
+                    if descr.loop_version():
+                        self.faildescrs.append(descr)
 
-    def compiled(self):
-        if self.operations is None:
-            # root version must always be compiled
-            return True
-
-        return self._compiled[0] is not None
-
-    def copy_operations(self, operations):
-        from rpython.jit.metainterp.compile import (ResumeGuardDescr,
-                CompileLoopVersionDescr)
-        ignore = (rop.DEBUG_MERGE_POINT,)
-        oplist = []
-        for op in operations:
-            if op.getopnum() in ignore:
-                continue
-            cloned = op.clone()
-            oplist.append(cloned)
-            if cloned.is_guard():
-                olddescr = cloned.getdescr()
-                if not olddescr:
-                    continue
-                descr = olddescr.clone()
-                cloned.setdescr(descr)
-                if olddescr.loop_version():
-                    # copy the version
-                    assert isinstance(olddescr, CompileLoopVersionDescr)
-                    assert isinstance(descr, CompileLoopVersionDescr)
-                    descr.version = olddescr.version
-                    self.faildescrs.append(descr)
-        return oplist
-
-    def register_guard(self, op, version):
-        from rpython.jit.metainterp.compile import CompileLoopVersionDescr
-        assert isinstance(op, GuardResOp)
-        descr = op.getdescr()
-        if not descr.loop_version():
-            assert 0, "cannot register a guard that is not a CompileLoopVersionDescr"
-        assert isinstance(descr, CompileLoopVersionDescr)
-        descr.version = version
+    def register_guard(self, op, descr, version):
+        assert descr.loop_version()
         self.faildescrs.append(descr)
         # note: stitching a guard must resemble the order of the label
         # otherwise a wrong mapping is handed to the register allocator
@@ -903,12 +874,9 @@
             return self.operations[index]
         return None
 
-    def snapshot(self):
-        if len(self.versions) == 0:
-            # create a root version, simplyfies the code in compile.py
-            self.versions.append(LoopVersion(None))
-        root_version = self.versions[0]
-        version = LoopVersion(self)
+    def snapshot(self, operations):
+        version = LoopVersion(operations)
+        version.setup_once()
         self.versions.append(version)
         return version
 
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -151,10 +151,10 @@
     def set_to_none(self, operations):
         assert operations[self.index] is self.op
         operations[self.index] = None
-        descr = self.op.getdescr()
-        if descr and descr.loop_version():
-            assert isinstance(descr, CompileLoopVersionDescr)
-            descr.version = None
+        #descr = self.op.getdescr()
+        #if descr and descr.loop_version():
+        #    assert isinstance(descr, CompileLoopVersionDescr)
+        #    descr.version = None
         if operations[self.index-1] is self.cmp_op:
             operations[self.index-1] = None
 
@@ -276,10 +276,11 @@
                     continue
                 descr = op.getdescr()
                 if descr.loop_version():
-                    root_version.register_guard(op, version)
+                    assert isinstance(descr, ResumeGuardDescr)
+                    root_version.register_guard(op, descr, version)
 
             if user_code:
-                version = loop.snapshot()
+                version = loop.snapshot(copy_operations(loop.operations))
                 self.eliminate_array_bound_checks(loop, root_version, version)
 
     def emit_operation(self, op):
@@ -302,7 +303,9 @@
                 transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
                     other.set_to_none(loop.operations)
-                    root_version.register_guard(transitive_guard, version)
+                    descr = transitive_guard.getdescr()
+                    assert isinstance(descr, ResumeGuardDescr)
+                    root_version.register_guard(transitive_guard, descr, version)
 
         if self.has_two_labels:
             oplist = [loop.operations[0]] + self._newoperations + \
@@ -312,3 +315,23 @@
             loop.operations = self._newoperations + \
                     [op for op in loop.operations if op]
 
+def copy_operations(operations):
+    ignore = (rop.DEBUG_MERGE_POINT,)
+    oplist = []
+    for op in operations:
+        if op.getopnum() in ignore:
+            continue
+        cloned = op.clone()
+        oplist.append(cloned)
+        if cloned.is_guard():
+            olddescr = cloned.getdescr()
+            if not olddescr:
+                continue
+            assert isinstance(olddescr, ResumeGuardDescr)
+            descr = olddescr.clone()
+            assert isinstance(descr, ResumeGuardDescr)
+            cloned.setdescr(descr)
+            if olddescr.loop_version():
+                # copy the version
+                descr.version = olddescr.version
+    return oplist
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -23,7 +23,7 @@
 from rpython.jit.metainterp.optimizeopt.schedule import (VecScheduleData,
         Scheduler, Pack, Pair, AccumPair, vectorbox_outof_box, getpackopnum,
         getunpackopnum, PackType, determine_input_output_types)
-from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
+from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt, copy_operations
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib import listsort
 from rpython.rlib.objectmodel import we_are_translated
@@ -42,7 +42,7 @@
     if user_code and user_loop_bail_fast_path(loop, warmstate):
         return
     # the original loop (output of optimize_unroll)
-    version = loop.snapshot()
+    version = loop.snapshot(copy_operations(loop.operations))
     try:
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")

From noreply at buildbot.pypy.org  Mon Aug 24 16:42:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 24 Aug 2015 16:42:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Issue #2123: fix this link, as a starter
Message-ID: <20150824144221.611DC1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79205:5d94037cb67e
Date: 2015-08-24 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/5d94037cb67e/

Log:	Issue #2123: fix this link, as a starter

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -352,8 +352,7 @@
 Except when otherwise stated (look for LICENSE files or copyright/license
 information at the beginning of each file) the files in the 'lib-python/2.7'
 directory are all copyrighted by the Python Software Foundation and licensed
-under the Python Software License of which you can find a copy here:
-http://www.python.org/doc/Copyright.html 
+under the terms that you can find here: https://docs.python.org/2/license.html
 
 License for 'pypy/module/unicodedata/'
 ======================================

From noreply at buildbot.pypy.org  Mon Aug 24 17:05:18 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Aug 2015 17:05:18 +0200 (CEST)
Subject: [pypy-commit] pypy default: future-proof another license link,
 code.google.com is going away (issue #2123)
Message-ID: <20150824150518.43FFA1C0400@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79206:585042437c1b
Date: 2015-08-24 18:05 +0300
http://bitbucket.org/pypy/pypy/changeset/585042437c1b/

Log:	future-proof another license link, code.google.com is going away
	(issue #2123)

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -434,4 +434,4 @@
 
 The code is based on gperftools. You may see a copy of the License for it at
 
-    https://code.google.com/p/gperftools/source/browse/COPYING
+    https://github.com/gperftools/gperftools/blob/master/COPYING

From noreply at buildbot.pypy.org  Mon Aug 24 17:51:08 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 17:51:08 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: do not store the version on the
 descr anymore, but on an external data structure just for this purpose
Message-ID: <20150824155108.5EF281C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79207:89a3655d9226
Date: 2015-08-24 17:34 +0200
http://bitbucket.org/pypy/pypy/changeset/89a3655d9226/

Log:	do not store the version on the descr anymore, but on an external
	data structure just for this purpose

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -194,30 +194,34 @@
     if loop.versions:
         # compile each version once for the first fail descr!
         # this assumes that the root trace (= loop) is already compiled
-        to_stitch = []
-        for version in loop.versions:
-            if not version.faildescrs:
+        compiled = {}
+        info = loop.version_info
+        for descr in info.descrs:
+            print "$>", descr
+            version = info.get(descr)
+            if not version:
+                # the guard might have been removed from the trace
                 continue
-            faildescr = version.faildescrs[0]
-            assert isinstance(faildescr, ResumeGuardDescr)
-            vl = create_empty_loop(metainterp)
-            vl.inputargs = version.inputargs
-            vl.operations = version.operations
-            vl.original_jitcell_token = jitcell_token
-            asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
-                                   faildescr, version.inputargs,
-                                   version.operations, jitcell_token)
-            record_loop_or_bridge(metainterp_sd, vl)
-            assert asminfo is not None
+            if version not in compiled:
+                print " +COMPILE", version
+                assert isinstance(descr, ResumeGuardDescr)
+                vl = create_empty_loop(metainterp)
+                vl.inputargs = version.inputargs
+                vl.operations = version.operations
+                vl.original_jitcell_token = jitcell_token
+                asminfo = send_bridge_to_backend(jitdriver_sd, metainterp_sd,
+                                                 descr, version.inputargs,
+                                                 version.operations, jitcell_token)
+                record_loop_or_bridge(metainterp_sd, vl)
+                assert asminfo is not None
+                compiled[version] = (asminfo, descr, version, jitcell_token)
+            else:
+                print " +stitch", version
+                param = compiled[version]
+                cpu.stitch_bridge(descr, param)
 
-            for i,fd in enumerate(version.faildescrs):
-                if i == 0:
-                    continue
-                to_stitch.append((fd, (asminfo, faildescr, version, jitcell_token)))
-        # stitch to the trace loop
-        for fd, param in to_stitch:
-            cpu.stitch_bridge(fd, param)
     loop.versions = None
+    loop.version_info = None
 
 def compile_retrace(metainterp, greenkey, start,
                     inputargs, jumpargs,
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -739,41 +739,72 @@
                 pass_by -= 1
     return -1
 
+class VersionInfo(object):
+    def __init__(self):
+        self.descrs = []
+        self.leads_to = {}
+        self.insert_index = -1
+
+    def mark(self):
+        self.insert_index = len(self.descrs)
+
+    def clear(self):
+        self.insert_index = -1
+
+    def track(self, op, descr, version):
+        #print "+++", descr, "=>", version
+        assert descr.loop_version()
+        if self.insert_index >= 0:
+            assert self.insert_index >= 0
+            self.descrs.insert(self.insert_index, descr)
+        else:
+            self.descrs.append(descr)
+        self.leads_to[descr] = version
+        # note: stitching a guard must resemble the order of the label
+        # otherwise a wrong mapping is handed to the register allocator
+        op.setfailargs(version.renamed_inputargs)
+        assert version.renamed_inputargs is not None
+
+    def remove(self, descr):
+        if descr in self.leads_to:
+            #print "---", descr, "=>", self.leads_to[descr]
+            del self.leads_to[descr]
+        else:
+            assert 0, "could not remove %s" % descr
+
+    def get(self, descr):
+        return self.leads_to.get(descr, None)
+
 class LoopVersion(object):
     """ A special version of a trace loop. Use loop.snaphost() to
         create one instance and attach it to a guard descr.
         If not attached to a descriptor, it will not be compiled.
     """
-    _compiled = (None,None,None,None)
     inputargs = None
     renamed_inputargs = None
 
     def __init__(self, operations):
-        self.faildescrs = None
-        self.stitchdescr = {}
         self.operations = operations
+        idx = index_of_first(rop.LABEL, self.operations)
+        assert idx >= 0
+        label = self.operations[idx]
+        self.inputargs = label.getarglist()
+        self.renamed_inputargs = label.getarglist()
 
-    def setup_once(self):
-        if self.operations:
-            idx = index_of_first(rop.LABEL, self.operations)
-            assert idx >= 0
-            label = self.operations[idx]
-            self.inputargs = label.getarglist()
-            self.renamed_inputargs = label.getarglist()
-            # register the faildescr for later stitching
-            for op in self.operations:
-                if op.is_guard():
-                    descr = op.getdescr()
-                    if descr.loop_version():
-                        self.faildescrs.append(descr)
-
-    def register_guard(self, op, descr, version):
-        assert descr.loop_version()
-        self.faildescrs.append(descr)
-        # note: stitching a guard must resemble the order of the label
-        # otherwise a wrong mapping is handed to the register allocator
-        op.setfailargs(version.renamed_inputargs)
-        assert version.renamed_inputargs is not None
+    def setup_once(self, info):
+        for op in self.operations:
+            if op.is_guard():
+                olddescr = op.getdescr()
+                if not olddescr:
+                    continue
+                descr = olddescr.clone()
+                op.setdescr(descr)
+                if descr.loop_version():
+                    toversion = info.leads_to.get(olddescr,None)
+                    if toversion:
+                        info.track(op, descr, toversion)
+                    else:
+                        assert 0, "olddescr must be found"
 
     def update_token(self, jitcell_token, all_target_tokens):
         # this is only invoked for versioned loops!
@@ -821,6 +852,7 @@
     def __init__(self, name):
         self.name = name
         self.versions = []
+        self.version_info = VersionInfo()
         # self.operations = list of ResOperations
         #   ops of the kind 'guard_xxx' contain a further list of operations,
         #   which may itself contain 'guard_xxx' and so on, making a tree.
@@ -858,6 +890,9 @@
         #
         self.operations = self.operations[:-1] + loop.operations
         self.versions = loop.versions
+        loop.versions = None
+        self.version_info = loop.version_info
+        loop.version_info = None
         if loop.quasi_immutable_deps:
             self.quasi_immutable_deps.update(loop.quasi_immutable_deps)
 
@@ -874,12 +909,24 @@
             return self.operations[index]
         return None
 
-    def snapshot(self, operations):
-        version = LoopVersion(operations)
-        version.setup_once()
+    def snapshot(self):
+        oplist = self.copy_operations(self.operations)
+        version = LoopVersion(oplist)
+        version.setup_once(self.version_info)
+        # register the faildescr for later stitching
         self.versions.append(version)
         return version
 
+    def copy_operations(self, operations):
+        ignore = (rop.DEBUG_MERGE_POINT,)
+        oplist = []
+        for op in operations:
+            if op.getopnum() in ignore:
+                continue
+            cloned = op.clone()
+            oplist.append(cloned)
+        return oplist
+
     def get_display_text(self):    # for graphpage.py
         return self.name + '\n' + repr(self.inputargs)
 
diff --git a/rpython/jit/metainterp/optimizeopt/guard.py b/rpython/jit/metainterp/optimizeopt/guard.py
--- a/rpython/jit/metainterp/optimizeopt/guard.py
+++ b/rpython/jit/metainterp/optimizeopt/guard.py
@@ -148,10 +148,12 @@
         self.setoperation(guard)
         self.setcmp(cmp_op)
 
-    def set_to_none(self, operations):
+    def set_to_none(self, loop):
+        operations = loop.operations
         assert operations[self.index] is self.op
         operations[self.index] = None
-        #descr = self.op.getdescr()
+        descr = self.op.getdescr()
+        loop.version_info.remove(descr)
         #if descr and descr.loop_version():
         #    assert isinstance(descr, CompileLoopVersionDescr)
         #    descr.version = None
@@ -266,22 +268,19 @@
         self.collect_guard_information(loop)
         self.eliminate_guards(loop)
         #
-        if len(loop.versions) >= 2:
-            assert len(loop.versions) == 2
-            root_version = loop.versions[0]
-            version = loop.versions[1]
+        assert len(loop.versions) == 1
+        version = loop.versions[0]
 
-            for op in loop.operations:
-                if not op.is_guard():
-                    continue
-                descr = op.getdescr()
-                if descr.loop_version():
-                    assert isinstance(descr, ResumeGuardDescr)
-                    root_version.register_guard(op, descr, version)
+        for i,op in enumerate(loop.operations):
+            if not op.is_guard():
+                continue
+            descr = op.getdescr()
+            if descr and descr.loop_version():
+                assert isinstance(descr, ResumeGuardDescr)
+                loop.version_info.track(op, descr, version)
 
-            if user_code:
-                version = loop.snapshot(copy_operations(loop.operations))
-                self.eliminate_array_bound_checks(loop, root_version, version)
+        if user_code:
+            self.eliminate_array_bound_checks(loop)
 
     def emit_operation(self, op):
         self.renamer.rename(op)
@@ -290,7 +289,10 @@
     def operation_position(self):
         return len(self._newoperations)
 
-    def eliminate_array_bound_checks(self, loop, root_version, version):
+    def eliminate_array_bound_checks(self, loop):
+        info = loop.version_info
+        info.mark()
+        version = None
         self._newoperations = []
         for key, guards in self.strongest_guards.items():
             if len(guards) <= 1:
@@ -302,10 +304,13 @@
             for other in guards[1:]:
                 transitive_guard = one.transitive_imply(other, self, loop)
                 if transitive_guard:
-                    other.set_to_none(loop.operations)
+                    if version is None:
+                        version = loop.snapshot()
+                    other.set_to_none(loop)
                     descr = transitive_guard.getdescr()
                     assert isinstance(descr, ResumeGuardDescr)
-                    root_version.register_guard(transitive_guard, descr, version)
+                    info.track(transitive_guard, descr, version)
+        info.clear()
 
         if self.has_two_labels:
             oplist = [loop.operations[0]] + self._newoperations + \
@@ -315,23 +320,3 @@
             loop.operations = self._newoperations + \
                     [op for op in loop.operations if op]
 
-def copy_operations(operations):
-    ignore = (rop.DEBUG_MERGE_POINT,)
-    oplist = []
-    for op in operations:
-        if op.getopnum() in ignore:
-            continue
-        cloned = op.clone()
-        oplist.append(cloned)
-        if cloned.is_guard():
-            olddescr = cloned.getdescr()
-            if not olddescr:
-                continue
-            assert isinstance(olddescr, ResumeGuardDescr)
-            descr = olddescr.clone()
-            assert isinstance(descr, ResumeGuardDescr)
-            cloned.setdescr(descr)
-            if olddescr.loop_version():
-                # copy the version
-                descr.version = olddescr.version
-    return oplist
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -23,7 +23,7 @@
 from rpython.jit.metainterp.optimizeopt.schedule import (VecScheduleData,
         Scheduler, Pack, Pair, AccumPair, vectorbox_outof_box, getpackopnum,
         getunpackopnum, PackType, determine_input_output_types)
-from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt, copy_operations
+from rpython.jit.metainterp.optimizeopt.guard import GuardStrengthenOpt
 from rpython.jit.metainterp.resoperation import (rop, ResOperation, GuardResOp)
 from rpython.rlib import listsort
 from rpython.rlib.objectmodel import we_are_translated
@@ -42,7 +42,7 @@
     if user_code and user_loop_bail_fast_path(loop, warmstate):
         return
     # the original loop (output of optimize_unroll)
-    version = loop.snapshot(copy_operations(loop.operations))
+    version = loop.snapshot()
     try:
         debug_start("vec-opt-loop")
         metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations, -2, None, None, "pre vectorize")
@@ -505,7 +505,7 @@
             return
         if vector:
             # add accumulation info to the descriptor
-            for version in self.loop.versions[1:]:
+            for version in self.loop.versions:
                 # this needs to be done for renamed (accum arguments)
                 version.renamed_inputargs = [ renamer.rename_map.get(arg,arg) for arg in version.inputargs ]
             self.appended_arg_count = len(sched_data.invariant_vector_vars)

From noreply at buildbot.pypy.org  Mon Aug 24 17:51:10 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 17:51:10 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: translation issue,
	insert index must be proven >= 0
Message-ID: <20150824155110.61A4E1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79208:b891a59cd1a7
Date: 2015-08-24 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/b891a59cd1a7/

Log:	translation issue, insert index must be proven >= 0

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -754,9 +754,10 @@
     def track(self, op, descr, version):
         #print "+++", descr, "=>", version
         assert descr.loop_version()
-        if self.insert_index >= 0:
-            assert self.insert_index >= 0
-            self.descrs.insert(self.insert_index, descr)
+        i = self.insert_index
+        if i >= 0:
+            assert i >= 0
+            self.descrs.insert(i, descr)
         else:
             self.descrs.append(descr)
         self.leads_to[descr] = version

From noreply at buildbot.pypy.org  Mon Aug 24 18:23:18 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 18:23:18 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: commented print
Message-ID: <20150824162318.9B6B41C12ED@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79209:ce56ac915e9a
Date: 2015-08-24 18:23 +0200
http://bitbucket.org/pypy/pypy/changeset/ce56ac915e9a/

Log:	commented print

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -197,13 +197,13 @@
         compiled = {}
         info = loop.version_info
         for descr in info.descrs:
-            print "$>", descr
+            #print "$>", descr
             version = info.get(descr)
             if not version:
                 # the guard might have been removed from the trace
                 continue
             if version not in compiled:
-                print " +COMPILE", version
+                #print " +COMPILE", version
                 assert isinstance(descr, ResumeGuardDescr)
                 vl = create_empty_loop(metainterp)
                 vl.inputargs = version.inputargs
@@ -216,7 +216,7 @@
                 assert asminfo is not None
                 compiled[version] = (asminfo, descr, version, jitcell_token)
             else:
-                print " +stitch", version
+                #print " +stitch", version
                 param = compiled[version]
                 cpu.stitch_bridge(descr, param)
 

From noreply at buildbot.pypy.org  Mon Aug 24 19:21:57 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 19:21:57 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: removed commented prints
Message-ID: <20150824172157.6BECA1C1192@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79210:eab397a1b567
Date: 2015-08-24 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/eab397a1b567/

Log:	removed commented prints

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -197,13 +197,11 @@
         compiled = {}
         info = loop.version_info
         for descr in info.descrs:
-            #print "$>", descr
             version = info.get(descr)
             if not version:
                 # the guard might have been removed from the trace
                 continue
             if version not in compiled:
-                #print " +COMPILE", version
                 assert isinstance(descr, ResumeGuardDescr)
                 vl = create_empty_loop(metainterp)
                 vl.inputargs = version.inputargs
@@ -216,7 +214,6 @@
                 assert asminfo is not None
                 compiled[version] = (asminfo, descr, version, jitcell_token)
             else:
-                #print " +stitch", version
                 param = compiled[version]
                 cpu.stitch_bridge(descr, param)
 
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -752,7 +752,6 @@
         self.insert_index = -1
 
     def track(self, op, descr, version):
-        #print "+++", descr, "=>", version
         assert descr.loop_version()
         i = self.insert_index
         if i >= 0:
@@ -768,7 +767,6 @@
 
     def remove(self, descr):
         if descr in self.leads_to:
-            #print "---", descr, "=>", self.leads_to[descr]
             del self.leads_to[descr]
         else:
             assert 0, "could not remove %s" % descr

From noreply at buildbot.pypy.org  Mon Aug 24 19:21:59 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 24 Aug 2015 19:21:59 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: updated llgraph to work with the
 new model (again :)
Message-ID: <20150824172159.C4FD41C1192@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79211:3742fae37b90
Date: 2015-08-24 19:21 +0200
http://bitbucket.org/pypy/pypy/changeset/3742fae37b90/

Log:	updated llgraph to work with the new model (again :)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -273,7 +273,7 @@
         self.vinfo_for_tests = kwds.get('vinfo_for_tests', None)
 
     def stitch_bridge(self, faildescr, target):
-        faildescr._llgraph_bridge = target._lltrace
+        faildescr._llgraph_bridge = target[0].lltrace
 
     def compile_loop(self, inputargs, operations, looptoken, jd_id=0,
                      unique_id=0, log=True, name='', logger=None):
@@ -292,8 +292,6 @@
         faildescr._llgraph_bridge = lltrace
         clt._llgraph_alltraces.append(lltrace)
         self._record_labels(lltrace)
-        if faildescr.loop_version():
-            faildescr.version._lltrace = lltrace
         return LLAsmInfo(lltrace)
 
     def _record_labels(self, lltrace):

From noreply at buildbot.pypy.org  Mon Aug 24 21:19:17 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 24 Aug 2015 21:19:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: no fcntl on win32,
 but now win32 leaks memory in these tests
Message-ID: <20150824191917.12BD21C0400@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79212:f5e65d929e0e
Date: 2015-08-24 22:18 +0300
http://bitbucket.org/pypy/pypy/changeset/f5e65d929e0e/

Log:	no fcntl on win32, but now win32 leaks memory in these tests

diff --git a/pypy/module/_multiprocessing/test/test_memory.py b/pypy/module/_multiprocessing/test/test_memory.py
--- a/pypy/module/_multiprocessing/test/test_memory.py
+++ b/pypy/module/_multiprocessing/test/test_memory.py
@@ -1,8 +1,12 @@
+import sys
+
 class AppTestMemory:
     spaceconfig = dict(usemodules=('_multiprocessing', 'mmap',
                                    '_rawffi', 'itertools',
-                                   'signal', 'select', 'fcntl',
+                                   'signal', 'select',
                                    'binascii'))
+    if sys.platform != 'win32':
+        spaceconfig['usemodules'] += ('fcntl',)
 
     def test_address_of(self):
         import _multiprocessing
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -1,12 +1,19 @@
+import sys
+
 from pypy.module._multiprocessing.interp_semaphore import (
     RECURSIVE_MUTEX, SEMAPHORE)
 
 
 class AppTestSemaphore:
     spaceconfig = dict(usemodules=('_multiprocessing', 'thread',
-                                   'signal', 'select', 'fcntl',
+                                   'signal', 'select',
                                    'binascii', 'struct'))
 
+    if sys.platform == 'win32':
+        spaceconfig['usemodules'] += ('_rawffi',)
+    else:
+        spaceconfig['usemodules'] += ('fcntl',)
+
     def setup_class(cls):
         cls.w_SEMAPHORE = cls.space.wrap(SEMAPHORE)
         cls.w_RECURSIVE = cls.space.wrap(RECURSIVE_MUTEX)

From noreply at buildbot.pypy.org  Tue Aug 25 09:29:23 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 09:29:23 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix a test
Message-ID: <20150825072923.E32CE1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79213:01cd081f0249
Date: 2015-08-24 14:02 +0200
http://bitbucket.org/pypy/pypy/changeset/01cd081f0249/

Log:	fix a test

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3660,7 +3660,7 @@
             return n
         res = self.meta_interp(f, [10])
         assert res == 0
-        self.check_resops({'int_gt': 2, 'getfield_gc': 1, 'int_eq': 1,
+        self.check_resops({'int_gt': 2, 'getfield_gc_i': 1, 'int_eq': 1,
                            'guard_true': 2, 'int_sub': 2, 'jump': 1,
                            'guard_false': 1})
 

From noreply at buildbot.pypy.org  Tue Aug 25 09:29:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 09:29:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: emit guards for inputargs too
Message-ID: <20150825072926.1885E1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79214:9f0c0da9727f
Date: 2015-08-25 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/9f0c0da9727f/

Log:	emit guards for inputargs too

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -369,8 +369,12 @@
 
     def use_box(self, box, preamble_op, optimizer=None):
         for arg in preamble_op.getarglist():
-            if isinstance(arg, Const) or isinstance(arg, AbstractInputArg):
-                pass
+            if isinstance(arg, Const):
+                continue
+            if isinstance(arg, AbstractInputArg):
+                info = arg.get_forwarded()
+                if info is not None and info is not empty_info:
+                    info.make_guards(arg, self.short)
             elif arg.get_forwarded() is None:
                 pass
             else:

From noreply at buildbot.pypy.org  Tue Aug 25 09:29:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 09:29:28 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: merge
Message-ID: <20150825072928.B8ABF1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79215:95d9c66d173c
Date: 2015-08-25 09:29 +0200
http://bitbucket.org/pypy/pypy/changeset/95d9c66d173c/

Log:	merge

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -90,10 +90,13 @@
 
 class SizeDescr(AbstractDescr):
     def __init__(self, S, vtable, runner):
+        assert not isinstance(vtable, bool)
         self.S = S
+        self._vtable = vtable
         self._is_object = vtable is not None
         self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
                                     get_field_descr=LLGraphCPU.fielddescrof)
+        self._runner = runner
 
     def get_all_fielddescrs(self):
         return self.all_fielddescrs
@@ -101,12 +104,12 @@
     def is_object(self):
         return self._is_object
 
-    def as_vtable_size_descr(self):
-        return self
-
     def get_vtable(self):
-        return heaptracker.adr2int(llmemory.cast_ptr_to_adr(
-            self._corresponding_vtable))
+        assert self._vtable is not None
+        if self._vtable is Ellipsis:
+            self._vtable = heaptracker.get_vtable_for_gcstruct(self._runner,
+                                                               self.S)
+        return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self._vtable))
 
     def count_fields_if_immutable(self):
         return heaptracker.count_fields_if_immutable(self.S)
@@ -413,11 +416,14 @@
     def sizeof(self, S, vtable):
         key = ('size', S)
         try:
-            return self.descrs[key]
+            descr = self.descrs[key]
         except KeyError:
             descr = SizeDescr(S, vtable, self)
             self.descrs[key] = descr
-            return descr
+        if descr._is_object and vtable is not Ellipsis:
+            assert vtable
+            heaptracker.testing_gcstruct2vtable.setdefault(S, vtable)
+        return descr
 
     def fielddescrof(self, S, fieldname):
         key = ('field', S, fieldname)
@@ -426,8 +432,7 @@
         except KeyError:
             descr = FieldDescr(S, fieldname)
             self.descrs[key] = descr
-            is_obj = heaptracker.has_gcstruct_a_vtable(S)
-            descr.parent_descr = self.sizeof(S, is_obj)
+            descr.parent_descr = self.sizeof(S, Ellipsis)
             if self.vinfo_for_tests is not None:
                 descr.vinfo = self.vinfo_for_tests
             return descr
@@ -695,7 +700,7 @@
         result = lltype.malloc(descr.S, zero=True)
         result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result)
         result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE,
-                                                descr._corresponding_vtable)
+                                                descr._vtable)
         return lltype.cast_opaque_ptr(llmemory.GCREF, result)
 
     def bh_new_array(self, length, arraydescr):
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -59,9 +59,6 @@
         return False
 
 class SizeDescrWithVTable(SizeDescr):
-    def as_vtable_size_descr(self):
-        return self
-
     def is_object(self):
         return True
 
@@ -198,10 +195,10 @@
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         if STRUCT is rclass.OBJECT:
-            fielddescr.parent_descr = get_size_descr(gccache, STRUCT, None)
+            vtable = None
         else:
-            fielddescr.parent_descr = get_size_descr(gccache, STRUCT,
-                            heaptracker.get_vtable_for_gcstruct(gccache, STRUCT))
+            vtable = heaptracker.get_vtable_for_gcstruct(gccache, STRUCT)
+        fielddescr.parent_descr = get_size_descr(gccache, STRUCT, vtable)
         return fielddescr
 
 def get_type_flag(TYPE):
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -121,7 +121,8 @@
             if op.getopnum() == rop.DEBUG_MERGE_POINT:
                 continue
             # ---------- GETFIELD_GC ----------
-            if op.getopnum() == rop.GETFIELD_GC:
+            if op.getopnum() in (rop.GETFIELD_GC_I, rop.GETFIELD_GC_F,
+                                 rop.GETFIELD_GC_R):
                 self.handle_getfield_gc(op)
                 continue
             # ---------- turn NEWxxx into CALL_MALLOC_xxx ----------
@@ -194,7 +195,7 @@
         followed by a bunch of 'setfields', and the 'pending_zeros'
         optimization we do here is meant for this case."""
         self.emit_pending_zeros()
-        self.newops.append(op)
+        self.emit_op(op)
 
     # ----------
 
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -8,7 +8,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.rtyper.lltypesystem import lltype, rffi, lltype, llmemory
 from rpython.rtyper import rclass
@@ -52,7 +51,6 @@
         O = lltype.GcStruct('O', ('parent', rclass.OBJECT),
                                  ('x', lltype.Signed))
         o_vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -7,7 +7,6 @@
 from rpython.jit.metainterp.gc import get_description
 from rpython.jit.tool.oparser import parse
 from rpython.jit.metainterp.optimizeopt.util import equaloplists
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.metainterp.history import JitCellToken, FLOAT
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -64,7 +63,6 @@
                                  ('x', lltype.Signed))
         o_descr = self.cpu.sizeof(O, True)
         o_vtable = globals()['o_vtable']
-        register_known_gctype(self.cpu, o_vtable, O)
         #
         tiddescr = self.gc_ll_descr.fielddescr_tid
         wbdescr = self.gc_ll_descr.write_barrier_descr
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -8,10 +8,6 @@
     total_freed_loops = 0
     total_freed_bridges = 0
 
-    # for heaptracker
-    # _all_size_descrs_with_vtable = None
-    _vtable_to_descr_dict = None
-
 class AbstractCPU(object):
     supports_floats = False
     supports_longlong = False
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -1820,7 +1820,6 @@
                 p = p.parent
         else:
             descr = cpu.sizeof(T, vtable_for_T)
-            descr._corresponding_vtable = vtable_for_T
         t = lltype.malloc(T)
         if T == self.T:
             t.parent.parent.typeptr = vtable_for_T
@@ -1950,8 +1949,7 @@
         cpu = self.cpu
         t_box, T_box, descr = self.alloc_instance(self.T)
         vtable = llmemory.cast_adr_to_ptr(
-            llmemory.cast_int_to_adr(T_box.getint()), heaptracker.VTABLETYPE)
-        heaptracker.register_known_gctype(cpu, vtable, self.T)
+            llmemory.cast_int_to_adr(T_box.getint()), rclass.CLASSTYPE)
         r1 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         r2 = self.execute_operation(rop.NEW_WITH_VTABLE, [], 'ref', descr)
         assert r1 != r2
diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py
--- a/rpython/jit/backend/test/test_ll_random.py
+++ b/rpython/jit/backend/test/test_ll_random.py
@@ -123,8 +123,6 @@
         vtable.name = rclass.alloc_array_name(name)
         self.structure_types_and_vtables.append((S, vtable))
         #
-        heaptracker.register_known_gctype(self.cpu, vtable, S)
-        #
         return S, vtable
 
     def get_random_structure(self, r, has_vtable=False):
diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py
--- a/rpython/jit/backend/x86/test/test_regalloc2.py
+++ b/rpython/jit/backend/x86/test/test_regalloc2.py
@@ -335,11 +335,7 @@
     cls1.subclassrange_min = 1
     cls1.subclassrange_max = 3
     if S is not None:
-        descr = cpu.sizeof(S)
-        if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-            cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._all_size_descrs_with_vtable.append(descr)
-        descr._corresponding_vtable = cls1
+        descr = cpu.sizeof(S, cls1)    # xxx may be not needed any more
     return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(cls1), "symbolic")
 
 def test_bug_2():
diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py
--- a/rpython/jit/codewriter/codewriter.py
+++ b/rpython/jit/codewriter/codewriter.py
@@ -74,7 +74,6 @@
             if not count % 500:
                 log.info("Produced %d jitcodes" % count)
         self.assembler.finished(self.callcontrol.callinfocollection)
-        heaptracker.finish_registering(self.cpu)
         log.info("there are %d JitCode instances." % count)
 
     def setup_vrefinfo(self, vrefinfo):
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -67,12 +67,14 @@
     if not has_gcstruct_a_vtable(GCSTRUCT):
         return None
     setup_cache_gcstruct2vtable(gccache)
-    return gccache._cache_gcstruct2vtable[GCSTRUCT]
+    try:
+        return gccache._cache_gcstruct2vtable[GCSTRUCT]
+    except KeyError:
+        return testing_gcstruct2vtable[GCSTRUCT]
 
 def setup_cache_gcstruct2vtable(gccache):
     if not hasattr(gccache, '_cache_gcstruct2vtable'):
         cache = {}
-        cache.update(testing_gcstruct2vtable)
         if gccache.rtyper:
             for rinstance in gccache.rtyper.instance_reprs.values():
                 cache[rinstance.lowleveltype.TO] = rinstance.rclass.getvtable()
@@ -88,57 +90,6 @@
 
 # ____________________________________________________________
 
-VTABLETYPE = rclass.CLASSTYPE
-
-def register_known_gctype(cpu, vtable, STRUCT):
-    # register the correspondance 'vtable' <-> 'STRUCT' in the cpu
-    sizedescr = cpu.sizeof(STRUCT, vtable)
-    assert sizedescr.as_vtable_size_descr() is sizedescr
-    if getattr(sizedescr, '_corresponding_vtable', None):
-        assert sizedescr._corresponding_vtable == vtable
-        return
-    assert lltype.typeOf(vtable) == VTABLETYPE
-    if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-        cpu.tracker._all_size_descrs_with_vtable = []
-        cpu.tracker._vtable_to_descr_dict = None
-    cpu.tracker._all_size_descrs_with_vtable.append(sizedescr)
-    sizedescr._corresponding_vtable = vtable
-
-def finish_registering(cpu):
-    # annotation hack for small examples which have no vtable at all
-    #if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'):
-    #    vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
-    #    register_known_gctype(cpu, vtable, rclass.OBJECT)
-    pass
-
-def vtable2descr(cpu, vtable):
-    assert lltype.typeOf(vtable) is lltype.Signed
-    vtable = int2adr(vtable)
-    if we_are_translated():
-        # Build the dict {vtable: sizedescr} at runtime.
-        # This is necessary because the 'vtables' are just pointers to
-        # static data, so they can't be used as keys in prebuilt dicts.
-        d = cpu.tracker._vtable_to_descr_dict
-        if d is None:
-            d = cpu.tracker._vtable_to_descr_dict = {}
-            for descr in cpu.tracker._all_size_descrs_with_vtable:
-                key = descr._corresponding_vtable
-                key = llmemory.cast_ptr_to_adr(key)
-                d[key] = descr
-        return d[vtable]
-    else:
-        vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE)
-        for descr in cpu.tracker._all_size_descrs_with_vtable:
-            if descr._corresponding_vtable == vtable:
-                return descr
-        raise KeyError(vtable)
-
-def descr2vtable(cpu, descr):
-    from rpython.jit.metainterp import history
-    assert isinstance(descr, history.AbstractDescr)
-    vtable = descr.as_vtable_size_descr()._corresponding_vtable
-    vtable = llmemory.cast_ptr_to_adr(vtable)
-    return adr2int(vtable)
 
 def all_fielddescrs(gccache, STRUCT, only_gc=False, res=None,
                     get_field_descr=None):
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -932,7 +932,6 @@
                     return self._do_builtin_call(op, 'alloc_with_del', [],
                                                  extra=(RESULT, vtable),
                                                  extrakey=STRUCT)
-            heaptracker.register_known_gctype(self.cpu, vtable, STRUCT)
             opname = 'new_with_vtable'
         else:
             opname = 'new'
@@ -1936,10 +1935,6 @@
     # VirtualRefs.
 
     def _handle_virtual_ref_call(self, op, oopspec_name, args):
-        vrefinfo = self.callcontrol.virtualref_info
-        heaptracker.register_known_gctype(self.cpu,
-                                          vrefinfo.jit_virtual_ref_vtable,
-                                          vrefinfo.JIT_VIRTUAL_REF)
         return SpaceOperation(oopspec_name, list(args), op.result)
 
     # -----------
diff --git a/rpython/jit/codewriter/test/test_codewriter.py b/rpython/jit/codewriter/test/test_codewriter.py
--- a/rpython/jit/codewriter/test/test_codewriter.py
+++ b/rpython/jit/codewriter/test/test_codewriter.py
@@ -25,8 +25,6 @@
 class FakeSizeDescr(AbstractDescr):
     def __init__(self, STRUCT):
         self.STRUCT = STRUCT
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeArrayDescr(AbstractDescr):
     def __init__(self, ARRAY):
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -39,8 +39,6 @@
         self.oopspecindex = oopspecindex
     def __repr__(self):
         return ''
-    def as_vtable_size_descr(self):
-        return self
 
 class FakeDict(object):
     def __getitem__(self, key):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -50,8 +50,7 @@
         return FakeDescr(('sizedescr', STRUCT))
 
 class FakeDescr(tuple):
-    def as_vtable_size_descr(self):
-        return self
+    pass
 
 class FakeLink:
     args = []
@@ -569,9 +568,6 @@
     op1 = Transformer(cpu).rewrite_operation(op)
     assert op1.opname == 'new_with_vtable'
     assert op1.args == [('sizedescr', S)]
-    #assert heaptracker.descr2vtable(cpu, op1.args[0]) == vtable [type check]
-    vtable_int = heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtable))
-    assert heaptracker.vtable2descr(cpu, vtable_int) == op1.args[0]
 
 def test_malloc_new_with_destructor():
     vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -242,6 +242,11 @@
         if self.size != -1:
             self.buffer = RawBuffer(cpu, None)
 
+    def _get_buffer(self):
+        buffer = self.buffer
+        assert buffer is not None
+        return buffer
+
     def getitem_raw(self, offset, itemsize, descr):
         if not self.is_virtual():
             raise InvalidRawOperation
@@ -249,37 +254,39 @@
             # for the test above: it's not enough to check is_virtual()
             # on the original object, because it might be a VRawSliceValue
             # instead.  If it is a virtual one, then we'll reach here anway.
-        return self.buffer.read_value(offset, itemsize, descr)
+        return self._get_buffer().read_value(offset, itemsize, descr)
 
     def setitem_raw(self, offset, itemsize, descr, itemop):
         if not self.is_virtual():
             raise InvalidRawOperation
-        self.buffer.write_value(offset, itemsize, descr, itemop)
+        self._get_buffer().write_value(offset, itemsize, descr, itemop)
 
     def is_virtual(self):
         return self.size != -1
 
     def _force_elements(self, op, optforce, descr):
         self.size = -1
-        for i in range(len(self.buffer.offsets)):
+        buffer = self._get_buffer()
+        for i in range(len(buffer.offsets)):
             # write the value
-            offset = self.buffer.offsets[i]
-            descr = self.buffer.descrs[i]
-            itembox = self.buffer.values[i]
+            offset = buffer.offsets[i]
+            descr = buffer.descrs[i]
+            itembox = buffer.values[i]
             setfield_op = ResOperation(rop.RAW_STORE,
                               [op, ConstInt(offset), itembox], descr=descr)
             optforce.emit_operation(setfield_op)
 
     def _visitor_walk_recursive(self, op, visitor, optimizer):
-        itemboxes = self.buffer.values
+        itemboxes = self._get_buffer().values
         visitor.register_virtual_fields(op, itemboxes)
         # there can be no virtuals stored in raw buffer
 
     @specialize.argtype(1)
     def visitor_dispatch_virtual_type(self, visitor):
+        buffer = self._get_buffer()
         return visitor.visit_vrawbuffer(self.size,
-                                        self.buffer.offsets[:],
-                                        self.buffer.descrs[:])
+                                        buffer.offsets[:],
+                                        buffer.descrs[:])
 
 class RawStructPtrInfo(AbstractRawPtrInfo):
     def __init__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -12,7 +12,6 @@
 from rpython.jit.metainterp.optimizeopt.util import sort_descrs, equaloplists
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.logger import LogOperations
-from rpython.jit.codewriter.heaptracker import register_known_gctype
 from rpython.jit.tool.oparser import OpParser
 from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr
 from rpython.jit.metainterp import compile, resume, history
@@ -122,9 +121,9 @@
     myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
     #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
-    nodesize = cpu.sizeof(NODE, True)
-    nodesize2 = cpu.sizeof(NODE2, True)
-    nodesize3 = cpu.sizeof(NODE3, True)
+    nodesize = cpu.sizeof(NODE, node_vtable)
+    nodesize2 = cpu.sizeof(NODE2, node_vtable2)
+    nodesize3 = cpu.sizeof(NODE3, node_vtable3)
     valuedescr = cpu.fielddescrof(NODE, 'value')
     floatdescr = cpu.fielddescrof(NODE, 'floatval')
     chardescr = cpu.fielddescrof(NODE, 'charval')
@@ -140,7 +139,7 @@
     QUASI = lltype.GcStruct('QUASIIMMUT', ('inst_field', lltype.Signed),
                             ('mutate_field', rclass.OBJECTPTR),
                             hints={'immutable_fields': accessor})
-    quasisize = cpu.sizeof(QUASI, False)
+    quasisize = cpu.sizeof(QUASI, None)
     quasi = lltype.malloc(QUASI, immortal=True)
     quasi.inst_field = -4247
     quasifielddescr = cpu.fielddescrof(QUASI, 'inst_field')
@@ -163,8 +162,8 @@
     intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval')
     immut_intval = cpu.fielddescrof(INTOBJ_IMMUT, 'intval')
-    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, True)
-    immut_descr = cpu.sizeof(INTOBJ_IMMUT, True)
+    noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, intobj_noimmut_vtable)
+    immut_descr = cpu.sizeof(INTOBJ_IMMUT, intobj_immut_vtable)
 
     PTROBJ_IMMUT = lltype.GcStruct('PTROBJ_IMMUT', ('parent', OBJECT),
                                             ('ptrval', lltype.Ptr(OBJECT)),
@@ -179,7 +178,7 @@
     # a GcStruct not inheriting from OBJECT
     S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed),
                         ('b', lltype.Ptr(NODE)))
-    ssize = cpu.sizeof(S, False)
+    ssize = cpu.sizeof(S, None)
     adescr = cpu.fielddescrof(S, 'a')
     abisdescr = cpu.fielddescrof(S, 'abis')
     bdescr = cpu.fielddescrof(S, 'b')
@@ -202,7 +201,7 @@
     inst_w_list = cpu.fielddescrof(W_ROOT, 'inst_w_list')
     w_root_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     
-    tsize = cpu.sizeof(T, False)
+    tsize = cpu.sizeof(T, None)
     cdescr = cpu.fielddescrof(T, 'c')
     ddescr = cpu.fielddescrof(T, 'd')
     arraydescr3 = cpu.arraydescrof(lltype.GcArray(lltype.Ptr(NODE)))
@@ -215,10 +214,10 @@
     SIMPLE = lltype.GcStruct('simple',
         ('parent', OBJECT),
         ('value', lltype.Signed))
-    simpledescr = cpu.sizeof(SIMPLE, True)
     simplevalue = cpu.fielddescrof(SIMPLE, 'value')
     simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
-    usize = cpu.sizeof(U, True)
+    simpledescr = cpu.sizeof(SIMPLE, simple_vtable)
+    #usize = cpu.sizeof(U, ...)
     onedescr = cpu.fielddescrof(U, 'one')
 
     FUNC = lltype.FuncType([lltype.Signed], lltype.Signed)
@@ -350,18 +349,7 @@
 
     jit_virtual_ref_vtable = vrefinfo.jit_virtual_ref_vtable
     jvr_vtable_adr = llmemory.cast_ptr_to_adr(jit_virtual_ref_vtable)
-    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, False)
-
-    register_known_gctype(cpu, node_vtable,  NODE)
-    register_known_gctype(cpu, node_vtable2, NODE2)
-    register_known_gctype(cpu, node_vtable3, NODE3)
-    register_known_gctype(cpu, u_vtable,     U)
-    register_known_gctype(cpu, simple_vtable,     SIMPLE)
-    register_known_gctype(cpu, jit_virtual_ref_vtable,vrefinfo.JIT_VIRTUAL_REF)
-    register_known_gctype(cpu, intobj_noimmut_vtable, INTOBJ_NOIMMUT)
-    register_known_gctype(cpu, intobj_immut_vtable,   INTOBJ_IMMUT)
-    register_known_gctype(cpu, ptrobj_immut_vtable,   PTROBJ_IMMUT)
-    register_known_gctype(cpu, w_root_vtable, W_ROOT)
+    vref_descr = cpu.sizeof(vrefinfo.JIT_VIRTUAL_REF, jit_virtual_ref_vtable)
 
     namespace = locals()
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py
--- a/rpython/jit/metainterp/optimizeopt/virtualize.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualize.py
@@ -1,5 +1,4 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
-from rpython.jit.codewriter.heaptracker import descr2vtable
 from rpython.jit.metainterp.history import ConstInt
 from rpython.jit.metainterp.history import CONST_NULL
 from rpython.jit.metainterp.optimizeopt import info, optimizer
@@ -199,7 +198,7 @@
             self.emit_operation(op)
 
     def optimize_NEW_WITH_VTABLE(self, op):
-        known_class = ConstInt(descr2vtable(self.optimizer.cpu, op.getdescr()))
+        known_class = ConstInt(op.getdescr().get_vtable())
         self.make_virtual(known_class, op, op.getdescr())
 
     def optimize_NEW(self, op):
diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py
--- a/rpython/jit/metainterp/test/test_compile.py
+++ b/rpython/jit/metainterp/test/test_compile.py
@@ -89,8 +89,6 @@
     metainterp.history = History()
     metainterp.history.operations = loop.operations[:-1]
     metainterp.history.inputargs = loop.inputargs[:]
-    cpu.tracker._all_size_descrs_with_vtable = (
-        LLtypeMixin.cpu.tracker._all_size_descrs_with_vtable)
     #
     greenkey = 'faked'
     target_token = compile_loop(metainterp, greenkey, 0,
diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py
--- a/rpython/jit/metainterp/test/test_warmspot.py
+++ b/rpython/jit/metainterp/test/test_warmspot.py
@@ -597,8 +597,7 @@
                 self._no = no
 
         class FakeDescr:
-            def as_vtable_size_descr(self):
-                return self
+            pass
 
         class FakeCPU(object):
             supports_floats = False

From noreply at buildbot.pypy.org  Tue Aug 25 09:57:14 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 09:57:14 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: disable the int_add collapsing,
 it's still buggy
Message-ID: <20150825075714.A41601C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79216:95eafd9acf17
Date: 2015-08-25 09:55 +0200
http://bitbucket.org/pypy/pypy/changeset/95eafd9acf17/

Log:	disable the int_add collapsing, it's still buggy

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -342,6 +342,7 @@
         return op
 
     def is_inputarg(self, op):
+        return True
         return op in self.inparg_dict
 
     def get_constant_box(self, box):

From noreply at buildbot.pypy.org  Tue Aug 25 09:57:16 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 09:57:16 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix
Message-ID: <20150825075716.BA0861C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79217:8cd118ebb7f9
Date: 2015-08-25 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/8cd118ebb7f9/

Log:	fix

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -700,7 +700,7 @@
         result = lltype.malloc(descr.S, zero=True)
         result_as_objptr = lltype.cast_pointer(rclass.OBJECTPTR, result)
         result_as_objptr.typeptr = support.cast_from_int(rclass.CLASSTYPE,
-                                                descr._vtable)
+                                                descr.get_vtable())
         return lltype.cast_opaque_ptr(llmemory.GCREF, result)
 
     def bh_new_array(self, length, arraydescr):

From noreply at buildbot.pypy.org  Tue Aug 25 10:38:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 10:38:57 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150825083857.A7A421C088E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79218:a0b1df551d4d
Date: 2015-08-25 10:38 +0200
http://bitbucket.org/pypy/pypy/changeset/a0b1df551d4d/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -305,6 +305,7 @@
     to the first operation.
     """
     from rpython.jit.metainterp.optimizeopt import optimize_trace
+    from rpython.jit.metainterp.optimizeopt.optimizer import BasicLoopInfo
 
     history = metainterp.history
     metainterp_sd = metainterp.staticdata
@@ -328,6 +329,7 @@
     except InvalidLoop:
         # Fall back on jumping to preamble
         xxx
+        return None
 
     loop = partial_trace
     loop.operations = (loop.operations + loop_info.extra_same_as +
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -123,9 +123,9 @@
         # the label_op again
         label_op.initarglist(label_args + extra)
         if new_virtual_state is not None:
-            res = self.jump_to_preamble(start_label.getdescr(), end_jump,
-                                         info)
-            xxx
+            self.jump_to_preamble(start_label.getdescr(), end_jump, info)
+            return (UnrollInfo(target_token, label_op, []),
+                    self.optimizer._newoperations)
             #return new_virtual_state, self.optimizer._newoperations
         return (UnrollInfo(target_token, label_op,
                            self.short_preamble_producer.extra_same_as),
diff --git a/rpython/jit/metainterp/test/test_loop.py b/rpython/jit/metainterp/test/test_loop.py
--- a/rpython/jit/metainterp/test/test_loop.py
+++ b/rpython/jit/metainterp/test/test_loop.py
@@ -657,8 +657,8 @@
             self.check_jitcell_token_count(1)   # 1 loop, callable from the interp
 
     def test_example(self):
-        myjitdriver = JitDriver(greens = [], #['i'],
-                                reds = ['i', 'res', 'a'])
+        myjitdriver = JitDriver(greens = ['i'],
+                                reds = ['res', 'a'])
         CO_INCREASE = 0
         CO_JUMP_BACK_3 = 1
         CO_DECREASE = 2

From noreply at buildbot.pypy.org  Tue Aug 25 10:44:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Aug 2015 10:44:34 +0200 (CEST)
Subject: [pypy-commit] pypy default: Attempt to clarify comment
Message-ID: <20150825084434.133B41C00BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79219:4563fae303c9
Date: 2015-08-25 10:44 +0200
http://bitbucket.org/pypy/pypy/changeset/4563fae303c9/

Log:	Attempt to clarify comment

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -631,9 +631,11 @@
         raise Exception("TreeLoop.token is killed")
     token = property(_token, _token)
 
-    # This is the jitcell where the trace starts. Labels within the trace might
-    # belong to some other jitcells in the sens that jumping to this other
-    # jitcell will result in a jump to the label.
+    # This is the jitcell where the trace starts.  Labels within the
+    # trace might belong to some other jitcells, i.e. they might have
+    # TargetTokens with a different value for 'targeting_jitcell_token'.
+    # But these TargetTokens also have a 'original_jitcell_token' field,
+    # which must be equal to this one.
     original_jitcell_token = None
 
     def __init__(self, name):

From noreply at buildbot.pypy.org  Tue Aug 25 11:01:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 11:01:53 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: whack at tests
Message-ID: <20150825090153.E43921C088E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79220:91e214d4653a
Date: 2015-08-25 11:01 +0200
http://bitbucket.org/pypy/pypy/changeset/91e214d4653a/

Log:	whack at tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -880,6 +880,7 @@
         """
         short = """
         [p1, p2]
+        guard_nonnull(p1) []
         i1 = getfield_gc_i(p1, descr=valuedescr)
         jump(i1)
         """
@@ -1284,7 +1285,7 @@
         preamble = """
         [i0, p1, p3]
         i28 = int_add(i0, 1)
-        i29 = int_add(i0, 2)
+        i29 = int_add(i28, 1)
         p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p30, descr=nextdescr)
         setfield_gc(p30, i28, descr=valuedescr)
@@ -1294,7 +1295,7 @@
         expected = """
         [i0, p1, p3]
         i28 = int_add(i0, 1)
-        i29 = int_add(i0, 2)
+        i29 = int_add(i28, 1)
         p30 = new_with_vtable(descr=nodesize)
         setfield_gc(p3, p30, descr=nextdescr)
         setfield_gc(p30, i28, descr=valuedescr)
@@ -2333,6 +2334,7 @@
         setfield_gc(p1, i4, descr=nextdescr)
         setarrayitem_gc(p3, 0, i5, descr=arraydescr)
         escape_n()
+        ifoo = arraylen_gc(p3, descr=arraydescr) # killed by the backend
         jump(p1, i1, i2, p3, i3)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -2577,6 +2579,7 @@
         expected = """
         [p1, i0, i6]
         i7 = int_add(i0, i6)
+        ifoo = arraylen_gc(p1, descr=arraydescr) # killed by the backend
         jump(p1, i7, i6)
         """
         self.optimize_loop(ops, expected)
@@ -3086,6 +3089,7 @@
         self.optimize_loop(ops, expected, preamble)
 
     def test_remove_multiple_add_1(self):
+        py.test.skip("disabled")
         ops = """
         [i0]
         i1 = int_add(i0, 1)
@@ -3103,6 +3107,7 @@
         self.optimize_loop(ops, expected)
 
     def test_remove_multiple_add_2(self):
+        py.test.skip("disabled")
         ops = """
         [i0]
         i1 = int_add(i0, 1)
@@ -3130,6 +3135,7 @@
         self.optimize_loop(ops, expected)
 
     def test_remove_multiple_add_3(self):
+        py.test.skip("disabled")
         ops = """
         [i0]
         i1 = int_add(i0, %s)
@@ -4665,6 +4671,7 @@
         p1 = new_array(i0, descr=gcarraydescr)
         i1 = arraylen_gc(p1)
         setarrayitem_gc(p0, 0, p1, descr=gcarraydescr)
+        ifoo = arraylen_gc(p0, descr=gcarraydescr)
         jump(i0, p0)
         """
         self.optimize_loop(ops, expected)

From noreply at buildbot.pypy.org  Tue Aug 25 11:04:14 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 11:04:14 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix tests
Message-ID: <20150825090414.74DB01C088E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79221:6d35fabfd54c
Date: 2015-08-25 11:04 +0200
http://bitbucket.org/pypy/pypy/changeset/6d35fabfd54c/

Log:	fix tests

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -6146,7 +6146,7 @@
     def test_immutable_constantfold_recursive(self):
         ops = """
         []
-        p0 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
+        p0 = new_with_vtable(descr=ptrobj_immut_descr)
         setfield_gc(p0, p0, descr=immut_ptrval)
         escape_n(p0)
         jump()
@@ -6160,6 +6160,8 @@
                 p1 = other.container.ptrval
                 p1cast = lltype.cast_pointer(lltype.Ptr(self.PTROBJ_IMMUT), p1)
                 return p1cast.ptrval == p1
+            def _normalizedcontainer(self):
+                return self
         self.namespace['ptrobjself'] = lltype._ptr(llmemory.GCREF,
                                                    PtrObjSelf())
         expected = """
@@ -6171,8 +6173,8 @@
         #
         ops = """
         []
-        p0 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
-        p1 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
+        p0 = new_with_vtable(descr=ptrobj_immut_descr)
+        p1 = new_with_vtable(descr=ptrobj_immut_descr)
         setfield_gc(p0, p1, descr=immut_ptrval)
         setfield_gc(p1, p0, descr=immut_ptrval)
         escape_n(p0)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -169,6 +169,7 @@
                                             ('ptrval', lltype.Ptr(OBJECT)),
                                             hints={'immutable': True})
     ptrobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
+    ptrobj_immut_descr = cpu.sizeof(PTROBJ_IMMUT, ptrobj_immut_vtable)
     immut_ptrval = cpu.fielddescrof(PTROBJ_IMMUT, 'ptrval')
 
     arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))

From noreply at buildbot.pypy.org  Tue Aug 25 12:52:35 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 12:52:35 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: implement
	force_at_the_end_of_preamble for now
Message-ID: <20150825105235.972661C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79222:d878ab104991
Date: 2015-08-25 12:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d878ab104991/

Log:	implement force_at_the_end_of_preamble for now

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -35,6 +35,11 @@
     def is_virtual(self):
         return False
 
+    def force_at_the_end_of_preamble(self, op, optforce, rec):
+        if not self.is_virtual():
+            return optforce.get_box_replacement(op)
+        return self._force_at_the_end_of_preamble(op, optforce, rec)
+
     def get_known_class(self, cpu):
         return None
 
@@ -115,6 +120,9 @@
             return newop
         return op
 
+    def _force_at_the_end_of_preamble(self, op, optforce, rec):
+        return self.force_box(op, optforce)
+
     def is_virtual(self):
         return self.vdescr is not None
 
@@ -170,6 +178,21 @@
                 else:
                     optforce.emit_operation(setfieldop)
 
+    def _force_at_the_end_of_preamble(self, op, optforce, rec):
+        if self._fields is None:
+            return optforce.get_box_replacement(op)
+        if self in rec:
+            return optforce.get_box_replacement(op)
+        rec[self] = None
+        for i, fldbox in enumerate(self._fields):
+            if fldbox is not None:
+                info = optforce.getptrinfo(fldbox)
+                if info is not None:
+                    fldbox = info.force_at_the_end_of_preamble(fldbox, optforce,
+                                                               rec)
+                    self._fields[i] = fldbox
+        return op
+
     def _visitor_walk_recursive(self, instbox, visitor, optimizer):
         lst = self.vdescr.get_all_fielddescrs()
         assert self.is_virtual()
@@ -425,6 +448,21 @@
                                            descr=descr)
             shortboxes.add_heap_op(op, getarrayitem_op)
 
+    def _force_at_the_end_of_preamble(self, op, optforce, rec):
+        if self._items is None:
+            return optforce.get_box_replacement(op)
+        if self in rec:
+            return optforce.get_box_replacement(op)
+        rec[self] = None
+        for i, fldbox in enumerate(self._items):
+            if fldbox is not None:
+                info = optforce.getptrinfo(fldbox)
+                if info is not None:
+                    fldbox = info.force_at_the_end_of_preamble(fldbox, optforce,
+                                                               rec)
+                    self._items[i] = fldbox
+        return op
+
     def make_guards(self, op, short):
         AbstractVirtualPtrInfo.make_guards(self, op, short)
         if self.lenbound is not None:
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -144,7 +144,7 @@
         return self.optimizer.get_box_replacement(op, not_const=not_const)
 
     def getlastop(self):
-        return self.optimizer._last_emitted_op
+        return self.optimizer.getlastop()
 
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         return self.optimizer.replace_op_with(op, newopnum, args, descr)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -193,8 +193,8 @@
 
     def jump_to_existing_trace(self, jump_op):
         jitcelltoken = jump_op.getdescr()
+        virtual_state = self.get_virtual_state(jump_op.getarglist())
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
-        virtual_state = self.get_virtual_state(args)
         infos = [self.optimizer.getinfo(arg) for arg in args]
         for target_token in jitcelltoken.target_tokens:
             target_virtual_state = target_token.virtual_state
@@ -202,8 +202,7 @@
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, jump_op.getarglist(), infos,
-                    self.optimizer.cpu)
+                    virtual_state, args, infos, self.optimizer.cpu)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
@@ -258,8 +257,8 @@
                 op.set_forwarded(None)
 
     def export_state(self, start_label, original_label_args, renamed_inputargs):
+        virtual_state = self.get_virtual_state(original_label_args)
         end_args = [self.get_box_replacement(a) for a in original_label_args]
-        virtual_state = self.get_virtual_state(end_args)
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -609,6 +609,9 @@
             pass
         if box.type == 'r':
             info = opt.getptrinfo(box)
+            if info is not None:
+                rec = {}
+                box = info.force_at_the_end_of_preamble(box, opt.optimizer, rec)
             if info is not None and info.is_virtual():
                 result = info.visitor_dispatch_virtual_type(self)
                 self.info[box] = result
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -502,17 +502,20 @@
 
     def make_vstring_plain(self, op, mode, length):
         vvalue = VStringPlainInfo(mode, True, length)
-        self.make_equal_to(op, vvalue)
+        op = self.replace_op_with(op, op.getopnum())
+        op.set_forwarded(vvalue)
         return vvalue
 
     def make_vstring_concat(self, op, mode, vleft, vright):
         vvalue = VStringConcatInfo(mode, vleft, vright, True)
-        self.make_equal_to(op, vvalue)
+        op = self.replace_op_with(op, op.getopnum())
+        op.set_forwarded(vvalue)
         return vvalue
 
     def make_vstring_slice(self, op, strbox, startbox, mode, lengthbox):
         vvalue = VStringSliceInfo(strbox, startbox, lengthbox, mode)
-        self.make_equal_to(op, vvalue)
+        op = self.replace_op_with(op, op.getopnum())
+        op.set_forwarded(vvalue)
         return vvalue
 
     def optimize_NEWSTR(self, op):

From noreply at buildbot.pypy.org  Tue Aug 25 12:52:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Tue, 25 Aug 2015 12:52:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: kill old code
Message-ID: <20150825105237.D5A571C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79223:80d60620e11e
Date: 2015-08-25 12:52 +0200
http://bitbucket.org/pypy/pypy/changeset/80d60620e11e/

Log:	kill old code

diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -114,12 +114,6 @@
         if self.lgtop is not None:
             return self.lgtop
         assert not self.is_virtual()
-        #if lengthop is not None:
-        #    xxx
-        #    box = self.force_box(op, string_optimizer)
-        #    lengthop = string_optimizer.optimizer.replace_op_with(lengthop,
-        #        mode.STRLEN, [box])
-        #else:
         if not create_ops:
             return None
         lengthop = ResOperation(mode.STRLEN, [op])
@@ -326,102 +320,6 @@
     def visitor_dispatch_virtual_type(self, visitor):
         return visitor.visit_vstrconcat(self.mode is mode_unicode)
 
-# class __extend__(optimizer.OptValue):
-#     """New methods added to the base class OptValue for this file."""
-
-#     def getstrlen(self, string_optimizer, mode, lengthop):
-#         if mode is mode_string:
-#             s = self.get_constant_string_spec(mode_string)
-#             if s is not None:
-#                 return ConstInt(len(s))
-#         else:
-#             s = self.get_constant_string_spec(mode_unicode)
-#             if s is not None:
-#                 return ConstInt(len(s))
-#         if string_optimizer is None:
-#             return None
-#         self.ensure_nonnull()
-#         box = self.force_box(string_optimizer)
-#         if lengthop is not None:
-#             lengthop = string_optimizer.optimizer.replace_op_with(lengthop,
-#                 mode.STRLEN, [box])
-#         else:
-#             lengthop = ResOperation(mode.STRLEN, [box])
-#         string_optimizer.emit_operation(lengthop)
-#         return lengthop
-
-#     @specialize.arg(1)
-#     def get_constant_string_spec(self, mode):
-#         if self.is_constant():
-#             s = self.box.getref(lltype.Ptr(mode.LLTYPE))
-#             return mode.hlstr(s)
-#         else:
-#             return None
-
-#     def string_copy_parts(self, string_optimizer, targetbox, offsetbox, mode):
-#         # Copies the pointer-to-string 'self' into the target string
-#         # given by 'targetbox', at the specified offset.  Returns the offset
-#         # at the end of the copy.
-#         lengthbox = self.getstrlen(string_optimizer, mode, None)
-#         srcbox = self.force_box(string_optimizer)
-#         return copy_str_content(string_optimizer, srcbox, targetbox,
-#                                 CONST_0, offsetbox, lengthbox, mode)
-
-
-
-class XVStringPlainInfo(object):
-    """A string built with newstr(const)."""
-
-    def _visitor_walk_recursive(self, visitor):
-        charboxes = []
-        for value in self._chars:
-            if value is not None:
-                box = value.get_key_box()
-            else:
-                box = None
-            charboxes.append(box)
-        visitor.register_virtual_fields(self.keybox, charboxes)
-        for value in self._chars:
-            if value is not None:
-                value.visitor_walk_recursive(visitor)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        return visitor.visit_vstrplain(self.mode is mode_unicode)
-
-
-class XVStringConcatInfo(object):
-    """The concatenation of two other strings."""
-
-    def _visitor_walk_recursive(self, visitor):
-        # we don't store the lengthvalue in guards, because the
-        # guard-failed code starts with a regular STR_CONCAT again
-        leftbox = self.left.get_key_box()
-        rightbox = self.right.get_key_box()
-        visitor.register_virtual_fields(self.keybox, [leftbox, rightbox])
-        self.left.visitor_walk_recursive(visitor)
-        self.right.visitor_walk_recursive(visitor)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        return visitor.visit_vstrconcat(self.mode is mode_unicode)
-
-
-class XVStringSliceInfo(object):
-
-    def _visitor_walk_recursive(self, visitor):
-        boxes = [self.vstr.get_key_box(),
-                 self.vstart.get_key_box(),
-                 self.vlength.get_key_box()]
-        visitor.register_virtual_fields(self.keybox, boxes)
-        self.vstr.visitor_walk_recursive(visitor)
-        self.vstart.visitor_walk_recursive(visitor)
-        self.vlength.visitor_walk_recursive(visitor)
-
-    @specialize.argtype(1)
-    def _visitor_dispatch_virtual_type(self, visitor):
-        return visitor.visit_vstrslice(self.mode is mode_unicode)
-
 
 def copy_str_content(string_optimizer, srcbox, targetbox,
                      srcoffsetbox, offsetbox, lengthbox, mode,

From noreply at buildbot.pypy.org  Tue Aug 25 13:23:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Aug 2015 13:23:12 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: * reimplement structs that are
 immutable constants, built during traces
Message-ID: <20150825112312.932821C13C2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79224:13fed87e2df5
Date: 2015-08-25 13:23 +0200
http://bitbucket.org/pypy/pypy/changeset/13fed87e2df5/

Log:	* reimplement structs that are immutable constants, built during
	traces

	* in llsupport/descr.py, merge SizeDescrWithVTable with SizeDescr

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -111,8 +111,8 @@
                                                                self.S)
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self._vtable))
 
-    def count_fields_if_immutable(self):
-        return heaptracker.count_fields_if_immutable(self.S)
+    def is_immutable(self):
+        return heaptracker.is_immutable_struct(self.S)
 
     def __repr__(self):
         return 'SizeDescr(%r)' % (self.S,)
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -36,37 +36,32 @@
     size = 0      # help translation
     tid = llop.combine_ushort(lltype.Signed, 0, 0)
     vtable = lltype.nullptr(rclass.OBJECT_VTABLE)
+    immutable_flag = False
 
-    def __init__(self, size, count_fields_if_immut=-1,
-                 gc_fielddescrs=None, all_fielddescrs=None,
-                 vtable=lltype.nullptr(rclass.OBJECT_VTABLE)):
+    def __init__(self, size, gc_fielddescrs=None, all_fielddescrs=None,
+                 vtable=lltype.nullptr(rclass.OBJECT_VTABLE),
+                 immutable_flag=False):
         self.size = size
-        self.count_fields_if_immut = count_fields_if_immut
         self.gc_fielddescrs = gc_fielddescrs
         self.all_fielddescrs = all_fielddescrs
         self.vtable = vtable
+        self.immutable_flag = immutable_flag
 
     def get_all_fielddescrs(self):
         return self.all_fielddescrs
 
-    def count_fields_if_immutable(self):
-        return self.count_fields_if_immut
-
     def repr_of_descr(self):
         return '' % self.size
 
     def is_object(self):
-        return False
+        return bool(self.vtable)
 
-class SizeDescrWithVTable(SizeDescr):
-    def is_object(self):
-        return True
+    def is_immutable(self):
+        return self.immutable_flag
 
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self.vtable))
 
-BaseSizeDescr = SizeDescr
-
 def get_size_descr(gccache, STRUCT, vtable):
     cache = gccache._cache_size
     assert not isinstance(vtable, bool)
@@ -74,16 +69,14 @@
         return cache[STRUCT]
     except KeyError:
         size = symbolic.get_size(STRUCT, gccache.translate_support_code)
-        count_fields_if_immut = heaptracker.count_fields_if_immutable(STRUCT)
+        immutable_flag = heaptracker.is_immutable_struct(STRUCT)
         gc_fielddescrs = heaptracker.gc_fielddescrs(gccache, STRUCT)
         if vtable:
             assert heaptracker.has_gcstruct_a_vtable(STRUCT)
-            sizedescr = SizeDescrWithVTable(size, count_fields_if_immut,
-                                            gc_fielddescrs, None, vtable)
         else:
             assert not heaptracker.has_gcstruct_a_vtable(STRUCT)
-            sizedescr = SizeDescr(size, count_fields_if_immut,
-                                  gc_fielddescrs, None)
+        sizedescr = SizeDescr(size, gc_fielddescrs, vtable=vtable,
+                              immutable_flag=immutable_flag)
         gccache.init_size_descr(STRUCT, sizedescr)
         cache[STRUCT] = sizedescr
         all_fielddescrs = heaptracker.all_fielddescrs(gccache, STRUCT)
@@ -124,7 +117,7 @@
         return 'FieldDescr<%s>' % (self.name,)
 
     def check_correct_type(self, struct):
-        if isinstance(self.parent_descr, SizeDescrWithVTable):
+        if self.parent_descr.is_object():
             cls = llmemory.cast_adr_to_ptr(
                 heaptracker.int2adr(self.parent_descr.get_vtable()),
                 lltype.Ptr(rclass.OBJECT_VTABLE))
diff --git a/rpython/jit/backend/llsupport/test/test_descr.py b/rpython/jit/backend/llsupport/test/test_descr.py
--- a/rpython/jit/backend/llsupport/test/test_descr.py
+++ b/rpython/jit/backend/llsupport/test/test_descr.py
@@ -13,24 +13,24 @@
     T = lltype.GcStruct('T')
     S = lltype.GcStruct('S', ('x', lltype.Char),
                              ('y', lltype.Ptr(T)))
-    descr_s = get_size_descr(c0, S, False)
-    descr_t = get_size_descr(c0, T, False)
+    descr_s = get_size_descr(c0, S, None)
+    descr_t = get_size_descr(c0, T, None)
     assert descr_s.size == symbolic.get_size(S, False)
     assert descr_t.size == symbolic.get_size(T, False)
-    assert descr_s.count_fields_if_immutable() == -1
-    assert descr_t.count_fields_if_immutable() == -1
+    assert descr_s.is_immutable() == False
+    assert descr_t.is_immutable() == False
     assert descr_t.gc_fielddescrs == []
     assert len(descr_s.gc_fielddescrs) == 1
-    assert descr_s == get_size_descr(c0, S, False)
-    assert descr_s != get_size_descr(c1, S, False)
+    assert descr_s == get_size_descr(c0, S, None)
+    assert descr_s != get_size_descr(c1, S, None)
     #
-    descr_s = get_size_descr(c1, S, False)
+    descr_s = get_size_descr(c1, S, None)
     assert isinstance(descr_s.size, Symbolic)
-    assert descr_s.count_fields_if_immutable() == -1
+    assert descr_s.is_immutable() == False
 
     PARENT = lltype.Struct('P', ('x', lltype.Ptr(T)))
     STRUCT = lltype.GcStruct('S', ('parent', PARENT), ('y', lltype.Ptr(T)))
-    descr_struct = get_size_descr(c0, STRUCT, False)
+    descr_struct = get_size_descr(c0, STRUCT, None)
     assert len(descr_struct.gc_fielddescrs) == 2
 
 def test_get_size_descr_immut():
@@ -46,11 +46,11 @@
                         ('miss1', lltype.Void),
                         ('miss2', lltype.Void),
                         hints={'immutable': True})
-    for STRUCT, expected in [(S, 0), (T, 1), (U, 3), (V, 3)]:
+    for STRUCT in [S, T, U, V]:
         for translated in [False, True]:
             c0 = GcCache(translated)
-            descr_s = get_size_descr(c0, STRUCT, False)
-            assert descr_s.count_fields_if_immutable() == expected
+            descr_s = get_size_descr(c0, STRUCT, None)
+            assert descr_s.is_immutable() == True
 
 def test_get_field_descr():
     U = lltype.Struct('U')
@@ -329,7 +329,7 @@
     S = lltype.GcStruct('S', ('x', lltype.Char),
                              ('y', lltype.Ptr(T)),
                              ('z', lltype.Ptr(T)))
-    descr1 = get_size_descr(c0, S, False)
+    descr1 = get_size_descr(c0, S, None)
     s = symbolic.get_size(S, False)
     assert repr_of_descr(descr1) == '' % s
     #
diff --git a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_pinned_object_rewrite.py
@@ -1,7 +1,7 @@
 from test_rewrite import get_size_descr, get_array_descr, get_description, BaseFakeCPU
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescrWithVTable, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework, MovableObjectTracker
 from rpython.jit.backend.llsupport import jitframe, gc
@@ -115,7 +115,7 @@
         #
         class FakeCPU(BaseFakeCPU):
             def sizeof(self, STRUCT, is_object):
-                descr = SizeDescrWithVTable(104)
+                descr = SizeDescr(104)
                 descr.tid = 9315
                 descr.vtable = 12
                 return descr
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -1,6 +1,6 @@
 from rpython.jit.backend.llsupport.descr import get_size_descr,\
      get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\
-     SizeDescrWithVTable, get_interiorfield_descr
+     SizeDescr, get_interiorfield_descr
 from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\
      GcLLDescr_framework
 from rpython.jit.backend.llsupport import jitframe
@@ -163,8 +163,8 @@
         class FakeCPU(BaseFakeCPU):
             def sizeof(self, STRUCT, is_object):
                 assert is_object
-                return SizeDescrWithVTable(102, gc_fielddescrs=[],
-                                           vtable=o_vtable)
+                return SizeDescr(102, gc_fielddescrs=[],
+                                 vtable=o_vtable)
         self.cpu = FakeCPU()
         self.gc_ll_descr = GcLLDescr_boehm(None, None, None)
 
@@ -301,7 +301,7 @@
         #
         class FakeCPU(BaseFakeCPU):
             def sizeof(self, STRUCT, is_object):
-                descr = SizeDescrWithVTable(104, gc_fielddescrs=[])
+                descr = SizeDescr(104, gc_fielddescrs=[])
                 descr.tid = 9315
                 return descr
         self.cpu = FakeCPU()
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -20,30 +20,8 @@
     a -= r_uint(1 << (b8 - 1))     # a -= 128
     return intmask(a)
 
-def count_fields_if_immutable(STRUCT):
-    if not isinstance(STRUCT, lltype.GcStruct):
-        return -1
-    if STRUCT._hints.get('immutable', False):
-        try:
-            return _count_fields(STRUCT)
-        except ValueError:
-            pass
-    return -1
-
-def _count_fields(STRUCT):
-    if STRUCT == rclass.OBJECT:
-        return 0    # don't count 'typeptr'
-    result = 0
-    for fieldname, TYPE in STRUCT._flds.items():
-        if TYPE is lltype.Void:
-            pass       # ignore Voids
-        elif not isinstance(TYPE, lltype.ContainerType):
-            result += 1
-        elif isinstance(TYPE, lltype.GcStruct):
-            result += _count_fields(TYPE)
-        else:
-            raise ValueError(TYPE)
-    return result
+def is_immutable_struct(S):
+    return isinstance(S, lltype.GcStruct) and S._hints.get('immutable', False)
 
 # ____________________________________________________________
 
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -5,6 +5,7 @@
 from rpython.jit.metainterp.history import ConstInt, Const
 from rpython.rtyper.lltypesystem import lltype
 from rpython.jit.metainterp.optimizeopt.rawbuffer import RawBuffer, InvalidRawOperation
+from rpython.jit.metainterp.executor import execute
 
 
 INFO_NULL = 0
@@ -22,7 +23,10 @@
     def getconst(self):
         raise Exception("not a constant")
 
-    
+    def _is_immutable_and_filled_with_constants(self, optimizer, memo=None):
+        return False
+
+
 class PtrInfo(AbstractInfo):
     _attrs_ = ()
 
@@ -109,6 +113,15 @@
     def force_box(self, op, optforce):
         if self.is_virtual():
             optforce.forget_numberings()
+            #
+            if self._is_immutable_and_filled_with_constants(optforce.optimizer):
+                constptr = optforce.optimizer.constant_fold(op)
+                op.set_forwarded(constptr)
+                descr = self.vdescr
+                self.vdescr = None
+                self._force_elements_immutable(descr, constptr, optforce)
+                return constptr
+            #
             op.set_forwarded(None)
             optforce._emit_operation(op)
             newop = optforce.getlastop()
@@ -214,6 +227,46 @@
         getfield_op = ResOperation(opnum, [structbox], descr=descr)
         shortboxes.add_heap_op(op, getfield_op)
 
+    def _is_immutable_and_filled_with_constants(self, optimizer, memo=None):
+        # check if it is possible to force the given structure into a
+        # compile-time constant: this is allowed only if it is declared
+        # immutable, if all fields are already filled, and if each field
+        # is either a compile-time constant or (recursively) a structure
+        # which also answers True to the same question.
+        #
+        assert self.is_virtual()
+        if not self.vdescr.is_immutable():
+            return False
+        if memo is not None and self in memo:
+            return True       # recursive case: assume yes
+        #
+        for op in self._fields:
+            if op is None:
+                return False     # there is an uninitialized field
+            op = op.get_box_replacement()
+            if op.is_constant():
+                pass            # it is a constant value: ok
+            else:
+                fieldinfo = optimizer.getptrinfo(op)
+                if fieldinfo and fieldinfo.is_virtual():
+                    # recursive check
+                    if memo is None:
+                        memo = {self: None}
+                    if not fieldinfo._is_immutable_and_filled_with_constants(
+                            optimizer, memo):
+                        return False
+                else:
+                    return False    # not a constant at all
+        return True
+
+    def _force_elements_immutable(self, descr, constptr, optforce):
+        for i, flddescr in enumerate(descr.get_all_fielddescrs()):
+            fld = self._fields[i]
+            subbox = optforce.force_box(fld)
+            assert isinstance(subbox, Const)
+            execute(optforce.optimizer.cpu, None, rop.SETFIELD_GC,
+                    flddescr, constptr, subbox)
+
 class InstancePtrInfo(AbstractStructPtrInfo):
     _attrs_ = ('_known_class',)
     _fields = None
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -6091,7 +6091,7 @@
         # ----------
         ops = """
         [p1]
-        p0 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
+        p0 = new_with_vtable(descr=ptrobj_immut_descr)
         setfield_gc(p0, p1, descr=immut_ptrval)
         escape_n(p0)
         jump(p1)
@@ -6100,8 +6100,8 @@
         # ----------
         ops = """
         []
-        p0 = new_with_vtable(ConstClass(ptrobj_immut_vtable))
-        p1 = new_with_vtable(ConstClass(intobj_immut_vtable))
+        p0 = new_with_vtable(descr=ptrobj_immut_descr)
+        p1 = new_with_vtable(descr=immut_descr)
         setfield_gc(p1, 1242, descr=immut_intval)
         setfield_gc(p0, p1, descr=immut_ptrval)
         escape_n(p0)
@@ -6115,6 +6115,8 @@
                 p1 = other.container.ptrval
                 p1cast = lltype.cast_pointer(lltype.Ptr(self.INTOBJ_IMMUT), p1)
                 return p1cast.intval == 1242
+            def _normalizedcontainer(self):
+                return self
         self.namespace['ptrobj1242'] = lltype._ptr(llmemory.GCREF,
                                                    PtrObj1242())
         expected = """
@@ -6191,6 +6193,8 @@
                 assert p2 != p1
                 p2cast = lltype.cast_pointer(lltype.Ptr(self.PTROBJ_IMMUT), p2)
                 return p2cast.ptrval == p1
+            def _normalizedcontainer(self):
+                return self
         self.namespace['ptrobjself2'] = lltype._ptr(llmemory.GCREF,
                                                     PtrObjSelf2())
         expected = """

From noreply at buildbot.pypy.org  Tue Aug 25 13:46:19 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 25 Aug 2015 13:46:19 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix
Message-ID: <20150825114619.D149F1C00BD@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79225:29a7fb10a98d
Date: 2015-08-25 13:46 +0200
http://bitbucket.org/pypy/pypy/changeset/29a7fb10a98d/

Log:	fix

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -432,7 +432,12 @@
         except KeyError:
             descr = FieldDescr(S, fieldname)
             self.descrs[key] = descr
-            descr.parent_descr = self.sizeof(S, Ellipsis)
+            if (isinstance(S, lltype.GcStruct) and
+                    heaptracker.has_gcstruct_a_vtable(S)):
+                vtable = Ellipsis
+            else:
+                vtable = None
+            descr.parent_descr = self.sizeof(S, vtable)
             if self.vinfo_for_tests is not None:
                 descr.vinfo = self.vinfo_for_tests
             return descr

From noreply at buildbot.pypy.org  Tue Aug 25 16:49:18 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Aug 2015 16:49:18 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: change signature of
	stm_hashtable_list()
Message-ID: <20150825144918.B161F1C05CE@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1944:a3cb98b78053
Date: 2015-08-25 16:53 +0200
http://bitbucket.org/pypy/stmgc/changeset/a3cb98b78053/

Log:	change signature of stm_hashtable_list()

	Storing GC references into raw memory can be problematic.

diff --git a/c8/stm/hashtable.c b/c8/stm/hashtable.c
--- a/c8/stm/hashtable.c
+++ b/c8/stm/hashtable.c
@@ -325,6 +325,9 @@
                 stm_allocate_preexisting(sizeof(stm_hashtable_entry_t),
                                          (char *)&initial.header);
             hashtable->additions++;
+            /* make sure .object is NULL in all segments before
+               "publishing" the entry in the hashtable: */
+            write_fence();
         }
         table->items[i] = entry;
         write_fence();     /* make sure 'table->items' is written here */
@@ -422,7 +425,7 @@
 }
 
 long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
-                        stm_hashtable_entry_t **results)
+                        stm_hashtable_entry_t * TLPREFIX *results)
 {
     /* Set the read marker.  It will be left as long as we're running
        the same transaction.
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -733,8 +733,13 @@
 void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
                                object_t *nvalue);
 long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+
+/* WARNING: stm_hashtable_list does not do a stm_write() on the 'results'
+   argument. 'results' may point inside an object. So if 'results' may be
+   a part of an old obj (which may have survived a minor GC), then make
+   sure to call stm_write() on the obj before calling this function. */
 long stm_hashtable_list(object_t *, stm_hashtable_t *,
-                        stm_hashtable_entry_t **results);
+                        stm_hashtable_entry_t * TLPREFIX *results);
 extern uint32_t stm_hashtable_entry_userdata;
 void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
                            void (object_t **));
diff --git a/c8/test/support.py b/c8/test/support.py
--- a/c8/test/support.py
+++ b/c8/test/support.py
@@ -210,11 +210,11 @@
 bool _check_hashtable_write(object_t *, stm_hashtable_t *, uintptr_t key,
                             object_t *nvalue, stm_thread_local_t *tl);
 long stm_hashtable_length_upper_bound(stm_hashtable_t *);
-long stm_hashtable_list(object_t *, stm_hashtable_t *,
-                        stm_hashtable_entry_t **results);
 uint32_t stm_hashtable_entry_userdata;
 void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
                            void trace(object_t **));
+long _stm_hashtable_list(object_t *o, stm_hashtable_t *h,
+                         object_t *entries);
 
 void _set_hashtable(object_t *obj, stm_hashtable_t *h);
 stm_hashtable_t *_get_hashtable(object_t *obj);
@@ -397,6 +397,16 @@
     return entry->object;
 }
 
+long _stm_hashtable_list(object_t *o, stm_hashtable_t *h,
+                         object_t *entries)
+{
+    if (entries != NULL)
+        return stm_hashtable_list(o, h,
+            (stm_hashtable_entry_t * TLPREFIX*)((stm_char*)entries+SIZEOF_MYOBJ));
+    return stm_hashtable_list(o, h, NULL);
+}
+
+
 void _set_queue(object_t *obj, stm_queue_t *q)
 {
     stm_char *field_addr = ((stm_char*)obj);
diff --git a/c8/test/test_hashtable.py b/c8/test/test_hashtable.py
--- a/c8/test/test_hashtable.py
+++ b/c8/test/test_hashtable.py
@@ -23,15 +23,17 @@
 def htitems(o):
     h = get_hashtable(o)
     upper_bound = lib.stm_hashtable_length_upper_bound(h)
-    entries = ffi.new("stm_hashtable_entry_t *[]", upper_bound)
-    count = lib.stm_hashtable_list(o, h, entries)
+    entries = stm_allocate_refs(upper_bound)
+    count = lib._stm_hashtable_list(o, h, entries)
     assert count <= upper_bound
-    return [(lib._get_entry_index(entries[i]),
-             lib._get_entry_object(entries[i])) for i in range(count)]
+
+    return [(lib._get_entry_index(ffi.cast("stm_hashtable_entry_t *", stm_get_ref(entries, i))),
+             lib._get_entry_object(ffi.cast("stm_hashtable_entry_t *", stm_get_ref(entries, i))))
+            for i in range(count)]
 
 def htlen(o):
     h = get_hashtable(o)
-    count = lib.stm_hashtable_list(o, h, ffi.NULL)
+    count = lib._stm_hashtable_list(o, h, ffi.NULL)
     return count
 
 

From noreply at buildbot.pypy.org  Tue Aug 25 16:53:02 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 25 Aug 2015 16:53:02 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: fix for stmdict.get_items_w() and
	similar
Message-ID: <20150825145302.33E2D1C0710@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8-gcc
Changeset: r79226:861d6cdfd881
Date: 2015-08-25 16:55 +0200
http://bitbucket.org/pypy/pypy/changeset/861d6cdfd881/

Log:	fix for stmdict.get_items_w() and similar

	Import a new stmgc version that changes the interface of
	stm_hashtable_list to accept a GC array. Before, we used a raw-array
	that at some point contained GC references, which is not good if a
	minor GC occurs and we read from it later.

diff --git a/pypy/module/pypystm/hashtable.py b/pypy/module/pypystm/hashtable.py
--- a/pypy/module/pypystm/hashtable.py
+++ b/pypy/module/pypystm/hashtable.py
@@ -61,30 +61,21 @@
 
     def keys_w(self, space):
         array, count = self.h.list()
-        try:
-            lst = [intmask(array[i].index) for i in range(count)]
-        finally:
-            self.h.freelist(array)
+        lst = [intmask(array[i].index) for i in range(count)]
         return space.newlist_int(lst)
 
     def values_w(self, space):
         array, count = self.h.list()
-        try:
-            lst_w = [cast_gcref_to_instance(W_Root, array[i].object)
-                     for i in range(count)]
-        finally:
-            self.h.freelist(array)
+        lst_w = [cast_gcref_to_instance(W_Root, array[i].object)
+                 for i in range(count)]
         return space.newlist(lst_w)
 
     def items_w(self, space):
         array, count = self.h.list()
-        try:
-            lst_w = [space.newtuple([
-                         space.wrap(intmask(array[i].index)),
-                         cast_gcref_to_instance(W_Root, array[i].object)])
-                     for i in range(count)]
-        finally:
-            self.h.freelist(array)
+        lst_w = [space.newtuple([
+            space.wrap(intmask(array[i].index)),
+            cast_gcref_to_instance(W_Root, array[i].object)])
+                 for i in range(count)]
         return space.newlist(lst_w)
 
 
diff --git a/pypy/module/pypystm/stmdict.py b/pypy/module/pypystm/stmdict.py
--- a/pypy/module/pypystm/stmdict.py
+++ b/pypy/module/pypystm/stmdict.py
@@ -157,49 +157,40 @@
 
     def get_length(self):
         array, count = self.h.list()
-        try:
-            total_length_times_two = 0
-            for i in range(count):
-                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
-                assert subarray
-                total_length_times_two += len(subarray)
-        finally:
-            self.h.freelist(array)
+        total_length_times_two = 0
+        for i in range(count):
+            subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+            assert subarray
+            total_length_times_two += len(subarray)
         return total_length_times_two >> 1
 
     def get_keys_values_w(self, offset):
         array, count = self.h.list()
-        try:
-            result_list_w = []
-            for i in range(count):
-                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
-                assert subarray
-                j = offset
-                limit = len(subarray)
-                while j < limit:
-                    w_item = cast_gcref_to_instance(W_Root, subarray[j])
-                    result_list_w.append(w_item)
-                    j += 2
-        finally:
-            self.h.freelist(array)
+        result_list_w = []
+        for i in range(count):
+            subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+            assert subarray
+            j = offset
+            limit = len(subarray)
+            while j < limit:
+                w_item = cast_gcref_to_instance(W_Root, subarray[j])
+                result_list_w.append(w_item)
+                j += 2
         return result_list_w
 
     def get_items_w(self, space):
         array, count = self.h.list()
-        try:
-            result_list_w = []
-            for i in range(count):
-                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
-                assert subarray
-                j = 0
-                limit = len(subarray)
-                while j < limit:
-                    w_key = cast_gcref_to_instance(W_Root, subarray[j])
-                    w_value = cast_gcref_to_instance(W_Root, subarray[j + 1])
-                    result_list_w.append(space.newtuple([w_key, w_value]))
-                    j += 2
-        finally:
-            self.h.freelist(array)
+        result_list_w = []
+        for i in range(count):
+            subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+            assert subarray
+            j = 0
+            limit = len(subarray)
+            while j < limit:
+                w_key = cast_gcref_to_instance(W_Root, subarray[j])
+                w_value = cast_gcref_to_instance(W_Root, subarray[j + 1])
+                result_list_w.append(space.newtuple([w_key, w_value]))
+                j += 2
         return result_list_w
 
     def len_w(self, space):
diff --git a/pypy/module/pypystm/stmset.py b/pypy/module/pypystm/stmset.py
--- a/pypy/module/pypystm/stmset.py
+++ b/pypy/module/pypystm/stmset.py
@@ -96,28 +96,22 @@
 
     def get_length(self):
         array, count = self.h.list()
-        try:
-            total_length = 0
-            for i in range(count):
-                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
-                assert subarray
-                total_length += len(subarray)
-        finally:
-            self.h.freelist(array)
+        total_length = 0
+        for i in range(count):
+            subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+            assert subarray
+            total_length += len(subarray)
         return total_length
 
     def get_items_w(self):
         array, count = self.h.list()
-        try:
-            result_list_w = []
-            for i in range(count):
-                subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
-                assert subarray
-                for j in range(len(subarray)):
-                    w_item = cast_gcref_to_instance(W_Root, subarray[j])
-                    result_list_w.append(w_item)
-        finally:
-            self.h.freelist(array)
+        result_list_w = []
+        for i in range(count):
+            subarray = lltype.cast_opaque_ptr(PARRAY, array[i].object)
+            assert subarray
+            for j in range(len(subarray)):
+                w_item = cast_gcref_to_instance(W_Root, subarray[j])
+                result_list_w.append(w_item)
         return result_list_w
 
     def len_w(self, space):
diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py
--- a/rpython/rlib/rstm.py
+++ b/rpython/rlib/rstm.py
@@ -214,7 +214,7 @@
                                        ('index', lltype.Unsigned),
                                        ('object', llmemory.GCREF))
 _STM_HASHTABLE_ENTRY_P = lltype.Ptr(_STM_HASHTABLE_ENTRY)
-_STM_HASHTABLE_ENTRY_ARRAY = rffi.CArray(_STM_HASHTABLE_ENTRY_P)
+_STM_HASHTABLE_ENTRY_ARRAY = lltype.GcArray(_STM_HASHTABLE_ENTRY_P)
 
 @dont_look_inside
 def _ll_hashtable_get(h, key):
@@ -234,17 +234,14 @@
 def _ll_hashtable_list(h):
     upper_bound = llop.stm_hashtable_length_upper_bound(lltype.Signed,
                                                         h.ll_raw_hashtable)
-    array = lltype.malloc(_STM_HASHTABLE_ENTRY_ARRAY, upper_bound,
-                          flavor='raw')
+    array = lltype.malloc(_STM_HASHTABLE_ENTRY_ARRAY, upper_bound)
+    # 'array' is newly allocated, thus we don't need to do a manual
+    # write_barrier for stm as requested by stm_hashtable_list
     count = llop.stm_hashtable_list(lltype.Signed, h, h.ll_raw_hashtable,
-                                    array)
+                                    lltype.direct_arrayitems(array))
     return (array, count)
 
 @dont_look_inside
-def _ll_hashtable_freelist(h, array):
-    lltype.free(array, flavor='raw')
-
- at dont_look_inside
 def _ll_hashtable_lookup(h, key):
     return llop.stm_hashtable_lookup(_STM_HASHTABLE_ENTRY_P,
                                      h, h.ll_raw_hashtable, key)
@@ -261,7 +258,6 @@
                                            'set': _ll_hashtable_set,
                                            'len': _ll_hashtable_len,
                                           'list': _ll_hashtable_list,
-                                      'freelist': _ll_hashtable_freelist,
                                         'lookup': _ll_hashtable_lookup,
                                       'writeobj': _ll_hashtable_writeobj})
 NULL_HASHTABLE = lltype.nullptr(_HASHTABLE_OBJ)
@@ -332,9 +328,6 @@
             items.append("additional garbage for testing")
         return items, count
 
-    def freelist(self, array):
-        pass
-
     def lookup(self, key):
         assert type(key) is int
         return EntryObjectForTest(self, key)
diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py
--- a/rpython/translator/stm/funcgen.py
+++ b/rpython/translator/stm/funcgen.py
@@ -360,7 +360,7 @@
     arg2 = funcgen.expr(op.args[2])
     result = funcgen.expr(op.result)
     return ('%s = stm_hashtable_list((object_t *)%s, %s, '
-            '(stm_hashtable_entry_t **)%s);' % (result, arg0, arg1, arg2))
+            '(stm_hashtable_entry_t * TLPREFIX*)(%s));' % (result, arg0, arg1, arg2))
 
 def stm_hashtable_tracefn(funcgen, op):
     arg0 = funcgen.expr(op.args[0])
diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-6666c6fd1aad
+a3cb98b78053
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -271,7 +271,7 @@
 /* ############# commit log ############# */
 
 
-void _dbg_print_commit_log()
+void _dbg_print_commit_log(void)
 {
     struct stm_commit_log_entry_s *cl = &commit_log_root;
 
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -309,7 +309,7 @@
 {
     assert(_finalization_state(obj) == 1);
     /* The call will add GCFLAG_VISITED recursively, thus bump state 1->2 */
-    mark_visit_possibly_new_object(obj, pseg);
+    mark_visit_possibly_overflow_object(obj, pseg);
 }
 
 static struct list_s *mark_finalize_step1(
@@ -431,7 +431,7 @@
     if (f != NULL && f->run_finalizers != NULL) {
         LIST_FOREACH_R(f->run_finalizers, object_t * /*item*/,
                        ({
-                           mark_visit_possibly_new_object(item, pseg);
+                           mark_visit_possibly_overflow_object(item, pseg);
                        }));
     }
 }
diff --git a/rpython/translator/stm/src_stm/stm/gcpage.c b/rpython/translator/stm/src_stm/stm/gcpage.c
--- a/rpython/translator/stm/src_stm/stm/gcpage.c
+++ b/rpython/translator/stm/src_stm/stm/gcpage.c
@@ -341,7 +341,7 @@
 }
 
 
-static void mark_visit_possibly_new_object(object_t *obj, struct stm_priv_segment_info_s *pseg)
+static void mark_visit_possibly_overflow_object(object_t *obj, struct stm_priv_segment_info_s *pseg)
 {
     /* if newly allocated object, we trace in segment_base, otherwise in
        the sharing seg0 */
@@ -464,7 +464,7 @@
         for (; modified < end; modified++) {
             if (modified->type == TYPE_POSITION_MARKER &&
                     modified->type2 != TYPE_MODIFIED_HASHTABLE)
-                mark_visit_possibly_new_object(modified->marker_object, pseg);
+                mark_visit_possibly_overflow_object(modified->marker_object, pseg);
         }
     }
 }
@@ -503,11 +503,11 @@
         struct stm_shadowentry_s *base = tl->shadowstack_base;
         while (current-- != base) {
             if ((((uintptr_t)current->ss) & 3) == 0) {
-                mark_visit_possibly_new_object(current->ss, pseg);
+                mark_visit_possibly_overflow_object(current->ss, pseg);
             }
         }
 
-        mark_visit_possibly_new_object(tl->thread_local_obj, pseg);
+        mark_visit_possibly_overflow_object(tl->thread_local_obj, pseg);
 
         tl = tl->next;
     } while (tl != stm_all_thread_locals);
@@ -517,7 +517,7 @@
     assert(get_priv_segment(0)->transaction_state == TS_NONE);
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (get_priv_segment(i)->transaction_state != TS_NONE) {
-            mark_visit_possibly_new_object(
+            mark_visit_possibly_overflow_object(
                 get_priv_segment(i)->threadlocal_at_start_of_transaction,
                 get_priv_segment(i));
 
diff --git a/rpython/translator/stm/src_stm/stm/hashtable.c b/rpython/translator/stm/src_stm/stm/hashtable.c
--- a/rpython/translator/stm/src_stm/stm/hashtable.c
+++ b/rpython/translator/stm/src_stm/stm/hashtable.c
@@ -325,6 +325,9 @@
                 stm_allocate_preexisting(sizeof(stm_hashtable_entry_t),
                                          (char *)&initial.header);
             hashtable->additions++;
+            /* make sure .object is NULL in all segments before
+               "publishing" the entry in the hashtable: */
+            write_fence();
         }
         table->items[i] = entry;
         write_fence();     /* make sure 'table->items' is written here */
@@ -422,7 +425,7 @@
 }
 
 long stm_hashtable_list(object_t *hobj, stm_hashtable_t *hashtable,
-                        stm_hashtable_entry_t **results)
+                        stm_hashtable_entry_t * TLPREFIX *results)
 {
     /* Set the read marker.  It will be left as long as we're running
        the same transaction.
diff --git a/rpython/translator/stm/src_stm/stm/pages.c b/rpython/translator/stm/src_stm/stm/pages.c
--- a/rpython/translator/stm/src_stm/stm/pages.c
+++ b/rpython/translator/stm/src_stm/stm/pages.c
@@ -28,7 +28,7 @@
     uint64_t ta = __sync_add_and_fetch(&pages_ctl.total_allocated,
                                        add_or_remove);
 
-    if (ta >= pages_ctl.total_allocated_bound)
+    if (UNLIKELY(ta >= pages_ctl.total_allocated_bound))
         pages_ctl.major_collection_requested = true;
 
     return ta;
diff --git a/rpython/translator/stm/src_stm/stm/queue.c b/rpython/translator/stm/src_stm/stm/queue.c
--- a/rpython/translator/stm/src_stm/stm/queue.c
+++ b/rpython/translator/stm/src_stm/stm/queue.c
@@ -437,7 +437,7 @@
             queue_entry_t *entry = seg->added_in_this_transaction;
 
             while (entry != NULL) {
-                mark_visit_possibly_new_object(entry->object, pseg);
+                mark_visit_possibly_overflow_object(entry->object, pseg);
                 entry = entry->next;
             }
         } TREE_LOOP_END;
diff --git a/rpython/translator/stm/src_stm/stm/smallmalloc.c b/rpython/translator/stm/src_stm/stm/smallmalloc.c
--- a/rpython/translator/stm/src_stm/stm/smallmalloc.c
+++ b/rpython/translator/stm/src_stm/stm/smallmalloc.c
@@ -8,9 +8,9 @@
 
 typedef struct {
     uint8_t sz;
-} fpsz_t;
+} full_page_size_t;
 
-static fpsz_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
+static full_page_size_t full_pages_object_size[PAGE_SMSIZE_END - PAGE_SMSIZE_START];
 /* ^^^ This array contains the size (in number of words) of the objects
    in the given page, provided it's a "full page of small objects".  It
    is 0 if it's not such a page, if it's fully free, or if it's in
@@ -19,7 +19,7 @@
    technically full yet, it will be very soon in this case).
 */
 
-static fpsz_t *get_fpsz(char *smallpage)
+static full_page_size_t *get_full_page_size(char *smallpage)
 {
     uintptr_t pagenum = (((char *)smallpage) - END_NURSERY_PAGE * 4096UL - stm_object_pages) / 4096;
     /* <= PAGE_SMSIZE_END because we may ask for it when there is no
@@ -118,7 +118,7 @@
 
         /* Succeeded: we have a page in 'smallpage' */
         *fl = smallpage->next;
-        get_fpsz((char *)smallpage)->sz = n;
+        get_full_page_size((char *)smallpage)->sz = n;
         return (char *)smallpage;
     }
 
@@ -126,12 +126,15 @@
        Maybe we can pick one from free_uniform_pages.
      */
     smallpage = free_uniform_pages;
-    if (smallpage != NULL) {
+    if (LIKELY(smallpage != NULL)) {
         if (UNLIKELY(!__sync_bool_compare_and_swap(&free_uniform_pages,
                                                    smallpage,
                                                    smallpage->nextpage)))
             goto retry;
 
+        /* got a new page: */
+        increment_total_allocated(4096);
+
         /* Succeeded: we have a page in 'smallpage', which is not
            initialized so far, apart from the 'nextpage' field read
            above.  Initialize it.
@@ -153,7 +156,7 @@
         *previous = NULL;
 
         /* The first slot is immediately returned */
-        get_fpsz((char *)smallpage)->sz = n;
+        get_full_page_size((char *)smallpage)->sz = n;
         return (char *)smallpage;
     }
 
@@ -174,8 +177,6 @@
 
     struct small_free_loc_s *result = *fl;
 
-    increment_total_allocated(size);
-
     if (UNLIKELY(result == NULL)) {
         char *addr = _allocate_small_slowpath(size);
         ((struct object_s*)addr)->stm_flags = 0;
@@ -270,7 +271,6 @@
         }
         else if (!_smallmalloc_sweep_keep(p)) {
             /* the location should be freed now */
-            increment_total_allocated(-szword*8);
 #ifdef STM_TESTS
             /* fill location with 0xdd in all segs except seg0 */
             int j;
@@ -300,6 +300,7 @@
             any_object_remaining = true;
         }
     }
+
     if (!any_object_remaining) {
         /* give page back to free_uniform_pages and thus make it
            inaccessible from all other segments again (except seg0) */
@@ -311,9 +312,14 @@
 
         ((struct small_free_loc_s *)baseptr)->nextpage = free_uniform_pages;
         free_uniform_pages = (struct small_free_loc_s *)baseptr;
+
+        /* gave the page back */
+        increment_total_allocated(-4096);
     }
     else if (!any_object_dying) {
-        get_fpsz(baseptr)->sz = szword;
+        /* this is still a full page. only in this case we set the
+           full_page_size again: */
+        get_full_page_size(baseptr)->sz = szword;
     }
     else {
         check_order_inside_small_page(page_free);
@@ -339,9 +345,9 @@
             if (*fl != NULL) {
                 /* the entry in full_pages_object_size[] should already be
                    szword.  We reset it to 0. */
-                fpsz_t *fpsz = get_fpsz((char *)*fl);
-                assert(fpsz->sz == szword);
-                fpsz->sz = 0;
+                full_page_size_t *full_page_size = get_full_page_size((char *)*fl);
+                assert(full_page_size->sz == szword);
+                full_page_size->sz = 0;
                 sweep_small_page(getbaseptr(*fl), *fl, szword);
                 *fl = NULL;
             }
@@ -351,7 +357,7 @@
         while (page != NULL) {
             /* for every page in small_page_lists: assert that the
                corresponding full_pages_object_size[] entry is 0 */
-            assert(get_fpsz((char *)page)->sz == 0);
+            assert(get_full_page_size((char *)page)->sz == 0);
             nextpage = page->nextpage;
             sweep_small_page(getbaseptr(page), page, szword);
             page = nextpage;
@@ -361,10 +367,10 @@
     /* process the really full pages, which are the ones which still
        have a non-zero full_pages_object_size[] entry */
     char *pageptr = uninitialized_page_stop;
-    fpsz_t *fpsz_start = get_fpsz(pageptr);
-    fpsz_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
-                                               PAGE_SMSIZE_START];
-    fpsz_t *fpsz;
+    full_page_size_t *fpsz_start = get_full_page_size(pageptr);
+    full_page_size_t *fpsz_end = &full_pages_object_size[PAGE_SMSIZE_END -
+                                                         PAGE_SMSIZE_START];
+    full_page_size_t *fpsz;
     for (fpsz = fpsz_start; fpsz < fpsz_end; fpsz++, pageptr += 4096) {
         uint8_t sz = fpsz->sz;
         if (sz != 0) {
diff --git a/rpython/translator/stm/src_stm/stmgc.h b/rpython/translator/stm/src_stm/stmgc.h
--- a/rpython/translator/stm/src_stm/stmgc.h
+++ b/rpython/translator/stm/src_stm/stmgc.h
@@ -733,8 +733,13 @@
 void stm_hashtable_write_entry(object_t *hobj, stm_hashtable_entry_t *entry,
                                object_t *nvalue);
 long stm_hashtable_length_upper_bound(stm_hashtable_t *);
+
+/* WARNING: stm_hashtable_list does not do a stm_write() on the 'results'
+   argument. 'results' may point inside an object. So if 'results' may be
+   a part of an old obj (which may have survived a minor GC), then make
+   sure to call stm_write() on the obj before calling this function. */
 long stm_hashtable_list(object_t *, stm_hashtable_t *,
-                        stm_hashtable_entry_t **results);
+                        stm_hashtable_entry_t * TLPREFIX *results);
 extern uint32_t stm_hashtable_entry_userdata;
 void stm_hashtable_tracefn(struct object_s *, stm_hashtable_t *,
                            void (object_t **));

From noreply at buildbot.pypy.org  Tue Aug 25 23:07:40 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Tue, 25 Aug 2015 23:07:40 +0200 (CEST)
Subject: [pypy-commit] pypy default: more freebsd fixes (David Naylor on
	pypy-dev)
Message-ID: <20150825210740.7211F1C0710@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79227:c2f97b8c2415
Date: 2015-08-26 00:05 +0300
http://bitbucket.org/pypy/pypy/changeset/c2f97b8c2415/

Log:	more freebsd fixes (David Naylor on pypy-dev)

diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -46,6 +46,7 @@
 
 _POSIX = os.name == "posix"
 _MS_WINDOWS = os.name == "nt"
+_FREEBSD = sys.platform.startswith('freebsd')
 _64BIT = "64bit" in host_platform.architecture()[0]
 
 
@@ -1080,8 +1081,11 @@
             return ctypes.util.find_library('c')
 
     libc_name = get_libc_name()     # Make sure the name is determined during import, not at runtime
+    if _FREEBSD:
+        RTLD_DEFAULT = -2  # see 
+        rtld_default_lib = ctypes.CDLL("RTLD_DEFAULT", handle=RTLD_DEFAULT, **load_library_kwargs)
     # XXX is this always correct???
-    standard_c_lib = ctypes.CDLL(get_libc_name(), **load_library_kwargs)
+    standard_c_lib = ctypes.CDLL(libc_name, **load_library_kwargs)
 
 # ____________________________________________
 
@@ -1174,7 +1178,10 @@
                 not_found.append(libname)
 
     if cfunc is None:
-        cfunc = get_on_lib(standard_c_lib, funcname)
+        if _FREEBSD and funcname in ('dlopen', 'fdlopen', 'dlsym', 'dlfunc', 'dlerror', 'dlclose'):
+            cfunc = get_on_lib(rtld_default_lib, funcname)
+        else:
+            cfunc = get_on_lib(standard_c_lib, funcname)
         # XXX magic: on Windows try to load the function from 'kernel32' too
         if cfunc is None and hasattr(ctypes, 'windll'):
             cfunc = get_on_lib(ctypes.windll.kernel32, funcname)
@@ -1493,18 +1500,17 @@
             _where_is_errno().contents.value = TLS.errno
 
     if ctypes:
-        if sys.platform == 'win32':
+        if _MS_WINDOWS:
             standard_c_lib._errno.restype = ctypes.POINTER(ctypes.c_int)
             def _where_is_errno():
                 return standard_c_lib._errno()
 
-        elif sys.platform.startswith('linux') or sys.platform == 'freebsd6':
+        elif sys.platform.startswith('linux'):
             standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int)
             def _where_is_errno():
                 return standard_c_lib.__errno_location()
 
-        elif any(plat in sys.platform
-                 for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')):
+        elif sys.platform == 'darwin' or _FREEBSD:
             standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int)
             def _where_is_errno():
                 return standard_c_lib.__error()

From noreply at buildbot.pypy.org  Tue Aug 25 23:41:51 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 25 Aug 2015 23:41:51 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix three cases where space.wrap() was
 called with a potentially non-ASCII string.
Message-ID: <20150825214151.445B71C088E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79228:aeafe30eac60
Date: 2015-08-25 12:35 +0200
http://bitbucket.org/pypy/pypy/changeset/aeafe30eac60/

Log:	Fix three cases where space.wrap() was called with a potentially
	non-ASCII string.

diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -143,7 +143,8 @@
     if interactiveconfig.runmodule:
         command = args.pop(0)
     for arg in args:
-        space.call_method(space.sys.get('argv'), 'append', space.wrap(arg))
+        space.call_method(space.sys.get('argv'), 'append',
+                          space.wrap_fsdecoded(arg))
 
     # load the source of the program given as command-line argument
     if interactiveconfig.runcommand:
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -61,8 +61,8 @@
                         rlocale.setlocale(rlocale.LC_CTYPE, '')
                     except rlocale.LocaleError:
                         pass
-                w_executable = space.fsdecode(space.wrapbytes(argv[0]))
-                w_argv = space.newlist([space.fsdecode(space.wrapbytes(s))
+                w_executable = space.wrap_fsdecoded(argv[0])
+                w_argv = space.newlist([space.wrap_fsdecoded(s)
                                         for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
                 exitcode = space.int_w(w_exitcode)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1050,6 +1050,9 @@
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
 
+    def wrap_fsdecoded(self, x):
+        return self.fsdecode(self.wrapbytes(x))
+
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -19,7 +19,8 @@
 def compilecode(space, source, filename, cmd='exec'):
     w = space.wrap
     w_code = space.builtin.call(
-        'compile', space.wrapbytes(source), w(filename), w(cmd), w(0), w(0))
+        'compile', space.wrapbytes(source), space.wrap_fsdecoded(filename),
+        w(cmd), w(0), w(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -44,7 +45,8 @@
 
         space.setitem(w_globals, w('__builtins__'), space.builtin)
         if filename is not None:
-            space.setitem(w_globals, w('__file__'), w(filename))
+            space.setitem(w_globals, w('__file__'),
+                          space.wrap_fsdecoded(filename))
 
         retval = pycode.exec_code(space, w_globals, w_globals)
         if eval:

From noreply at buildbot.pypy.org  Tue Aug 25 23:41:54 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 25 Aug 2015 23:41:54 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: hg merge py3k
Message-ID: <20150825214154.612E31C088E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79229:c9a245e856b7
Date: 2015-08-25 12:38 +0200
http://bitbucket.org/pypy/pypy/changeset/c9a245e856b7/

Log:	hg merge py3k

diff too long, truncating to 2000 out of 3074 lines

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.0
+Version: 1.2.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.0"
-__version_info__ = (1, 2, 0)
+__version__ = "1.2.1"
+__version_info__ = (1, 2, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -46,7 +46,7 @@
 # endif
 #else
 # include 
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py
--- a/pypy/bin/pyinteractive.py
+++ b/pypy/bin/pyinteractive.py
@@ -143,7 +143,8 @@
     if interactiveconfig.runmodule:
         command = args.pop(0)
     for arg in args:
-        space.call_method(space.sys.get('argv'), 'append', space.wrap(arg))
+        space.call_method(space.sys.get('argv'), 'append',
+                          space.wrap_fsdecoded(arg))
 
     # load the source of the program given as command-line argument
     if interactiveconfig.runcommand:
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.6.1.rst
    whatsnew-2.6.0.rst
    whatsnew-2.5.1.rst
    whatsnew-2.5.0.rst
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -0,0 +1,72 @@
+========================
+What's new in PyPy 2.6.1
+========================
+
+.. this is a revision shortly after release-2.6.0
+.. startrev: 91904d5c5188
+
+.. branch: use_min_scalar
+Correctly resolve the output dtype of ufunc(array, scalar) calls.
+
+.. branch: stdlib-2.7.10
+
+Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+additional task into translation
+
+.. branch: int-float-list-strategy
+
+Use a compact strategy for Python lists that mix integers and floats,
+at least if the integers fit inside 32 bits.  These lists are now
+stored as an array of floats, like lists that contain only floats; the
+difference is that integers are stored as tagged NaNs.  (This should
+have no visible effect!  After ``lst = [42, 42.5]``, the value of
+``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
+
+.. branch: cffi-callback-onerror
+Part of cffi 1.2.
+
+.. branch: cffi-new-allocator
+Part of cffi 1.2.
+
+.. branch: unicode-dtype
+
+Partial implementation of unicode dtype and unicode scalars.
+
+.. branch: dtypes-compatability
+
+Improve compatibility with numpy dtypes; handle offsets to create unions,
+fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
+allow subclassing dtype
+
+.. branch: indexing
+
+Refactor array indexing to support ellipses.
+
+.. branch: numpy-docstrings
+
+Allow the docstrings of built-in numpy objects to be set at run-time.
+
+.. branch: nditer-revisited
+
+Implement nditer 'buffered' flag and fix some edge cases
+
+.. branch: ufunc-reduce
+
+Allow multiple axes in ufunc.reduce()
+
+.. branch: fix-tinylang-goals
+
+Update tinylang goals to match current rpython
+
+.. branch: vmprof-review
+
+Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,68 +2,6 @@
 What's new in PyPy 2.6+
 =======================
 
-.. this is a revision shortly after release-2.6.0
-.. startrev: 91904d5c5188
+.. this is a revision shortly after release-2.6.1
+.. startrev: 83ebc73d4fcb
 
-.. branch: use_min_scalar
-Correctly resolve the output dtype of ufunc(array, scalar) calls.
-
-.. branch: stdlib-2.7.10
-
-Update stdlib to version 2.7.10
-
-.. branch: issue2062
-
-.. branch: disable-unroll-for-short-loops
-The JIT no longer performs loop unrolling if the loop compiles to too much code.
-
-.. branch: run-create_cffi_imports
-
-Build cffi import libraries as part of translation by monkey-patching an 
-additional task into translation
-
-.. branch: int-float-list-strategy
-
-Use a compact strategy for Python lists that mix integers and floats,
-at least if the integers fit inside 32 bits.  These lists are now
-stored as an array of floats, like lists that contain only floats; the
-difference is that integers are stored as tagged NaNs.  (This should
-have no visible effect!  After ``lst = [42, 42.5]``, the value of
-``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
-
-.. branch: cffi-callback-onerror
-.. branch: cffi-new-allocator
-
-.. branch: unicode-dtype
-
-Partial implementation of unicode dtype and unicode scalars.
-
-.. branch: dtypes-compatability
-
-Improve compatibility with numpy dtypes; handle offsets to create unions,
-fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
-allow subclassing dtype
-
-.. branch: indexing
-
-Refactor array indexing to support ellipses.
-
-.. branch: numpy-docstrings
-
-Allow the docstrings of built-in numpy objects to be set at run-time.
-
-.. branch: nditer-revisited
-
-Implement nditer 'buffered' flag and fix some edge cases
-
-.. branch: ufunc-reduce
-
-Allow multiple axes in ufunc.reduce()
-
-.. branch: fix-tinylang-goals
-
-Update tinylang goals to match current rpython
-
-.. branch: vmprof-review
-
-Clean up of vmprof, notably to handle correctly multiple threads
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -61,8 +61,8 @@
                         rlocale.setlocale(rlocale.LC_CTYPE, '')
                     except rlocale.LocaleError:
                         pass
-                w_executable = space.fsdecode(space.wrapbytes(argv[0]))
-                w_argv = space.newlist([space.fsdecode(space.wrapbytes(s))
+                w_executable = space.wrap_fsdecoded(argv[0])
+                w_argv = space.newlist([space.wrap_fsdecoded(s)
                                         for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
                 exitcode = space.int_w(w_exitcode)
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1052,6 +1052,9 @@
         from pypy.objspace.std.listobject import make_empty_list_with_size
         return make_empty_list_with_size(self, sizehint)
 
+    def wrap_fsdecoded(self, x):
+        return self.fsdecode(self.wrapbytes(x))
+
     @jit.unroll_safe
     def exception_match(self, w_exc_type, w_check_class):
         """Checks if the given exception type matches 'w_check_class'."""
diff --git a/pypy/interpreter/main.py b/pypy/interpreter/main.py
--- a/pypy/interpreter/main.py
+++ b/pypy/interpreter/main.py
@@ -19,7 +19,8 @@
 def compilecode(space, source, filename, cmd='exec'):
     w = space.wrap
     w_code = space.builtin.call(
-        'compile', space.wrapbytes(source), w(filename), w(cmd), w(0), w(0))
+        'compile', space.wrapbytes(source), space.wrap_fsdecoded(filename),
+        w(cmd), w(0), w(0))
     pycode = space.interp_w(eval.Code, w_code)
     return pycode
 
@@ -44,7 +45,8 @@
 
         space.setitem(w_globals, w('__builtins__'), space.builtin)
         if filename is not None:
-            space.setitem(w_globals, w('__file__'), w(filename))
+            space.setitem(w_globals, w('__file__'),
+                          space.wrap_fsdecoded(filename))
 
         retval = pycode.exec_code(space, w_globals, w_globals)
         if eval:
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.0"
+VERSION = "1.2.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3427,4 +3427,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "1.2.0"
+    assert __version__ == "1.2.1"
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -21,11 +21,12 @@
             i = 0
             count = 0
             i += 5 * WORD # header
-            assert s[i] == 4
-            i += 1 # marker
-            assert s[i] == 4
-            i += 1 # length
-            i += len('pypy')
+            assert s[i    ] == 5    # MARKER_HEADER
+            assert s[i + 1] == 0    # 0
+            assert s[i + 2] == 1    # VERSION_THREAD_ID
+            assert s[i + 3] == 4    # len('pypy')
+            assert s[i + 4: i + 8] == b'pypy'
+            i += 8
             while i < len(s):
                 if s[i] == 3:
                     break
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -42,7 +42,7 @@
 }
 
 
-""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read())
+""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read())
 
 class TestDirect(object):
     def test_infrastructure(self):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -364,6 +364,8 @@
     PyObject *ht_name, *ht_slots;
 } PyHeapTypeObject;
 
+#define PyObject_Bytes PyObject_Str
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW	1	/* No string quotes etc. */
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -160,6 +160,35 @@
     assert func.name == 'sin'
     assert func.BType == ', ), , False>'
 
+def test_remove_line_continuation_comments():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        double // blah \\
+                  more comments
+        x(void);
+        double // blah\\\\
+        y(void);
+        double // blah\\ \
+                  etc
+        z(void);
+    """)
+    m = ffi.dlopen(lib_m)
+    m.x
+    m.y
+    m.z
+
+def test_line_continuation_in_defines():
+    ffi = FFI(backend=FakeBackend())
+    ffi.cdef("""
+        #define ABC\\
+            42
+        #define BCD   \\
+            43
+    """)
+    m = ffi.dlopen(lib_m)
+    assert m.ABC == 42
+    assert m.BCD == 43
+
 def test_define_not_supported_for_now():
     ffi = FFI(backend=FakeBackend())
     e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"')
@@ -238,6 +267,13 @@
     ffi = FFI()
     ffi.cdef("typedef _Bool bool; void f(bool);")
 
+def test_void_renamed_as_only_arg():
+    ffi = FFI()
+    ffi.cdef("typedef void void_t1;"
+             "typedef void_t1 void_t;"
+             "typedef int (*func_t)(void_t);")
+    assert ffi.typeof("func_t").args == ()
+
 def test_win_common_types():
     from cffi.commontypes import COMMON_TYPES, _CACHE
     from cffi.commontypes import win_common_types, resolve_common_type
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,6 +25,9 @@
     if 1:     # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
+    else:
+        kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
+                                      ['-Werror'])
     return recompiler._verify(ffi, module_name, source, *args, **kwds)
 
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py
@@ -318,15 +318,32 @@
                 import cffi
                 ffi = cffi.FFI()
                 ffi.set_source("pack3.mymod", "/*code would be here*/")
+                ffi._hi_there = 42
             """)
         with open("setup.py", "w") as f:
-            f.write("""if 1:
+            f.write("from __future__ import print_function\n"
+                """if 1:
                 from setuptools import setup
+                from distutils.command.build_ext import build_ext
+                import os
+
+                class TestBuildExt(build_ext):
+                    def pre_run(self, ext, ffi):
+                        print('_make_setuptools_api: in pre_run:', end=" ")
+                        assert ffi._hi_there == 42
+                        assert ext.name == "pack3.mymod"
+                        fn = os.path.join(os.path.dirname(self.build_lib),
+                                          '..', 'see_me')
+                        print('creating %r' % (fn,))
+                        open(fn, 'w').close()
+
                 setup(name='example1',
                       version='0.1',
                       packages=['pack3'],
                       package_dir={'': 'src1'},
-                      cffi_modules=["src1/pack3/_build.py:ffi"])
+                      cffi_modules=["src1/pack3/_build.py:ffi"],
+                      cmdclass={'build_ext': TestBuildExt},
+                      )
             """)
 
     @chdir_to_tmp
@@ -335,6 +352,7 @@
         self.run(["setup.py", "build"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None}}})
 
@@ -344,6 +362,7 @@
         self.run(["setup.py", "build_ext", "-i"])
         self.check_produced_files({'setup.py': None,
                                    'build': '?',
+                                   'see_me': None,
                                    'src1': {'pack3': {'__init__.py': None,
                                                       '_build.py': None,
                                                       'mymod.SO': None}}})
diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py
--- a/rpython/flowspace/objspace.py
+++ b/rpython/flowspace/objspace.py
@@ -13,6 +13,11 @@
 
 def _assert_rpythonic(func):
     """Raise ValueError if ``func`` is obviously not RPython"""
+    try:
+        func.func_code.co_cellvars
+    except AttributeError:
+        raise ValueError("%r is not RPython: it is likely an unexpected "
+                         "built-in function or type" % (func,))
     if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'):
         raise ValueError("%r is tagged as NOT_RPYTHON" % (func,))
     if func.func_code.co_cellvars:
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -1363,6 +1363,15 @@
         simplify_graph(graph)
         assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1}
 
+    def test_unexpected_builtin_function(self):
+        import itertools
+        e = py.test.raises(ValueError, build_flow, itertools.permutations)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, itertools.tee)
+        assert ' is not RPython:' in str(e.value)
+        e = py.test.raises(ValueError, build_flow, Exception.__init__)
+        assert ' is not RPython:' in str(e.value)
+
 
 DATA = {'x': 5,
         'y': 6}
diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py
--- a/rpython/jit/backend/arm/runner.py
+++ b/rpython/jit/backend/arm/runner.py
@@ -64,12 +64,6 @@
                                               operations,
                                               original_loop_token, log=log)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU_ARM.cast_adr_to_int(adr)
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -636,8 +636,7 @@
             assert isinstance(box, Box)
             loc = self.fm.get_new_loc(box)
             locs.append(loc.value - base_ofs)
-        if looptoken.compiled_loop_token is not None:
-            # for tests
+        if looptoken.compiled_loop_token is not None:   # <- for tests
             looptoken.compiled_loop_token._ll_initial_locs = locs
 
     def can_merge_with_next_guard(self, op, i, operations):
diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c
--- a/rpython/jit/backend/llsupport/src/codemap.c
+++ b/rpython/jit/backend/llsupport/src/codemap.c
@@ -6,9 +6,9 @@
 #endif
 
 #ifdef RPYTHON_VMPROF
-RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored);
+RPY_EXTERN void vmprof_ignore_signals(int ignored);
 static void pypy_codemap_invalid_set(int ignored) {
-    rpython_vmprof_ignore_signals(ignored);
+    vmprof_ignore_signals(ignored);
 }
 #else
 static void pypy_codemap_invalid_set(int ignored) {
diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py
--- a/rpython/jit/backend/x86/runner.py
+++ b/rpython/jit/backend/x86/runner.py
@@ -100,12 +100,6 @@
         return self.assembler.assemble_bridge(faildescr, inputargs, operations,
                                               original_loop_token, log, logger)
 
-    def clear_latest_values(self, count):
-        setitem = self.assembler.fail_boxes_ptr.setitem
-        null = lltype.nullptr(llmemory.GCREF.TO)
-        for index in range(count):
-            setitem(index, null)
-
     def cast_ptr_to_int(x):
         adr = llmemory.cast_ptr_to_adr(x)
         return CPU386.cast_adr_to_int(adr)
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -268,6 +268,7 @@
                 y -= 1
             return res
         res = self.meta_interp(f, [6, sys.maxint, 48])
+        self.check_trace_count(6)
         assert res == f(6, sys.maxint, 48)
 
     def test_loop_invariant_mul_bridge_ovf2(self):
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -1087,6 +1087,16 @@
     """
     assert type(value) is cls
 
+def ll_record_exact_class(ll_value, ll_cls):
+    from rpython.rlib.debug import ll_assert
+    from rpython.rtyper.lltypesystem.lloperation import llop
+    from rpython.rtyper.lltypesystem import lltype
+    from rpython.rtyper.rclass import ll_type
+    ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument")
+    ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments")
+    llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls)
+
+
 class Entry(ExtRegistryEntry):
     _about_ = record_exact_class
 
@@ -1100,12 +1110,10 @@
         from rpython.rtyper import rclass
 
         classrepr = rclass.get_type_repr(hop.rtyper)
-
-        hop.exception_cannot_occur()
         v_inst = hop.inputarg(hop.args_r[0], arg=0)
         v_cls = hop.inputarg(classrepr, arg=1)
-        return hop.genop('jit_record_exact_class', [v_inst, v_cls],
-                         resulttype=lltype.Void)
+        hop.exception_is_here()
+        return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls)
 
 def _jit_conditional_call(condition, function, *args):
     pass
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -40,24 +40,20 @@
         **eci_kwds))
 
 
-    vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP,
-                                  compilation_info=eci)
-    vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT,
+    vmprof_init = rffi.llexternal("vmprof_init",
+                                  [rffi.INT, rffi.DOUBLE, rffi.CCHARP],
+                                  rffi.CCHARP, compilation_info=eci)
+    vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT,
+    vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT,
                                      compilation_info=eci,
                                      save_err=rffi.RFFI_SAVE_ERRNO)
-    vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf",
-                                       [rffi.CCHARP, rffi.LONG],
-                                       lltype.Void, compilation_info=eci)
-
-    ## vmprof_register_virtual_function = rffi.llexternal(
-    ##     "vmprof_register_virtual_function",
-    ##     [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void,
-    ##     compilation_info=eci, _nowrapper=True)
-
-    vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals",
+    vmprof_register_virtual_function = rffi.llexternal(
+                                           "vmprof_register_virtual_function",
+                                           [rffi.CCHARP, rffi.LONG, rffi.INT],
+                                           rffi.INT, compilation_info=eci)
+    vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals",
                                             [rffi.INT], lltype.Void,
                                             compilation_info=eci)
     return CInterface(locals())
@@ -83,10 +79,20 @@
 
     cont_name = 'rpyvmprof_f_%s_%s' % (name, token)
     tramp_name = 'rpyvmprof_t_%s_%s' % (name, token)
+    orig_tramp_name = tramp_name
 
     func.c_name = cont_name
     func._dont_inline_ = True
 
+    if sys.platform == 'darwin':
+        # according to internet "At the time UNIX was written in 1974...."
+        # "... all C functions are prefixed with _"
+        cont_name = '_' + cont_name
+        tramp_name = '_' + tramp_name
+        PLT = ""
+    else:
+        PLT = "@PLT"
+
     assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), (
         "rvmprof only supports x86-64 CPUs for now")
 
@@ -111,17 +117,15 @@
     target.write("""\
 \t.text
 \t.globl\t%(tramp_name)s
-\t.type\t%(tramp_name)s, @function
 %(tramp_name)s:
 \t.cfi_startproc
 \tpushq\t%(reg)s
 \t.cfi_def_cfa_offset 16
-\tcall %(cont_name)s at PLT
+\tcall %(cont_name)s%(PLT)s
 \taddq\t$8, %%rsp
 \t.cfi_def_cfa_offset 8
 \tret
 \t.cfi_endproc
-\t.size\t%(tramp_name)s, .-%(tramp_name)s
 """ % locals())
 
     def tok2cname(tok):
@@ -133,7 +137,7 @@
 
     header = 'RPY_EXTERN %s %s(%s);\n' % (
         tok2cname(restok),
-        tramp_name,
+        orig_tramp_name,
         ', '.join([tok2cname(tok) for tok in token] + ['long']))
 
     header += """\
@@ -147,7 +151,7 @@
 #endif
 #define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s
 }
-""" % (tramp_name, tramp_name, tramp_name)
+""" % (tramp_name, orig_tramp_name, tramp_name)
 
     eci = ExternalCompilationInfo(
         post_include_bits = [header],
@@ -155,7 +159,7 @@
     )
 
     return rffi.llexternal(
-        tramp_name,
+        orig_tramp_name,
         [token2lltype(tok) for tok in token] + [lltype.Signed],
         token2lltype(restok),
         compilation_info=eci,
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -1,14 +1,12 @@
 import sys, os
 from rpython.rlib.objectmodel import specialize, we_are_translated
-from rpython.rlib.rstring import StringBuilder
 from rpython.rlib import jit, rgc, rposix
 from rpython.rlib.rvmprof import cintf
 from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance
 from rpython.rtyper.lltypesystem import rffi
 
-MAX_CODES = 8000 - 255
-MAX_FUNC_NAME = 255
+MAX_FUNC_NAME = 1023
 
 # ____________________________________________________________
 
@@ -34,8 +32,6 @@
 
     def _cleanup_(self):
         self.is_enabled = False
-        self.fileno = -1
-        self._current_codes = None
 
     @specialize.argtype(1)
     def register_code(self, code, full_name_func):
@@ -102,18 +98,13 @@
         assert fileno >= 0
         if self.is_enabled:
             raise VMProfError("vmprof is already enabled")
-        if not (1e-6 <= interval < 1.0):
-            raise VMProfError("bad value for 'interval'")
-        interval_usec = int(interval * 1000000.0)
 
-        p_error = self.cintf.vmprof_init(fileno)
+        p_error = self.cintf.vmprof_init(fileno, interval, "pypy")
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
-        self.fileno = fileno
-        self._write_header(interval_usec)
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable(interval_usec)
+        res = self.cintf.vmprof_enable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
@@ -125,9 +116,6 @@
         if not self.is_enabled:
             raise VMProfError("vmprof is not enabled")
         self.is_enabled = False
-        if self._current_codes is not None:
-            self._flush_codes()
-        self.fileno = -1
         res = self.cintf.vmprof_disable()
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
@@ -136,48 +124,8 @@
         assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, (
             "the name must be 'class:func_name:func_line:filename' "
             "and at most %d characters; got '%s'" % (MAX_FUNC_NAME, name))
-        b = self._current_codes
-        if b is None:
-            b = self._current_codes = StringBuilder()
-        b.append('\x02')
-        _write_long_to_string_builder(uid, b)
-        _write_long_to_string_builder(len(name), b)
-        b.append(name)
-        if b.getlength() >= MAX_CODES:
-            self._flush_codes()
-
-    def _flush_codes(self):
-        buf = self._current_codes.build()
-        self._current_codes = None
-        self.cintf.vmprof_write_buf(buf, len(buf))
-        # NOTE: keep in mind that vmprof_write_buf() can only write
-        # a maximum of 8184 bytes.  This should be guaranteed here because:
-        assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184
-
-    def _write_header(self, interval_usec):
-        b = StringBuilder()
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(3, b)
-        _write_long_to_string_builder(0, b)
-        _write_long_to_string_builder(interval_usec, b)
-        _write_long_to_string_builder(0, b)
-        b.append('\x04') # interp name
-        b.append(chr(len('pypy')))
-        b.append('pypy')
-        buf = b.build()
-        self.cintf.vmprof_write_buf(buf, len(buf))
-
-
-def _write_long_to_string_builder(l, b):
-    b.append(chr(l & 0xff))
-    b.append(chr((l >> 8) & 0xff))
-    b.append(chr((l >> 16) & 0xff))
-    b.append(chr((l >> 24) & 0xff))
-    if sys.maxint > 2147483647:
-        b.append(chr((l >> 32) & 0xff))
-        b.append(chr((l >> 40) & 0xff))
-        b.append(chr((l >> 48) & 0xff))
-        b.append(chr((l >> 56) & 0xff))
+        if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0:
+            raise VMProfError("vmprof buffers full!  disk full or too slow")
 
 
 def vmprof_execute_code(name, get_code_fn, result_class=None):
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -1,22 +1,3 @@
-/* VMPROF
- *
- * statistical sampling profiler specifically designed to profile programs
- * which run on a Virtual Machine and/or bytecode interpreter, such as Python,
- * etc.
- *
- * The logic to dump the C stack traces is partly stolen from the code in
- * gperftools.
- * The file "getpc.h" has been entirely copied from gperftools.
- *
- * Tested only on gcc, linux, x86_64.
- *
- * Copyright (C) 2014-2015
- *   Antonio Cuni - anto.cuni at gmail.com
- *   Maciej Fijalkowski - fijall at gmail.com
- *   Armin Rigo - arigo at tunes.org
- *
- */
-
 #define _GNU_SOURCE 1
 
 
@@ -39,431 +20,4 @@
 #endif
 
 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include 
-#include "rvmprof_getpc.h"
-#include "rvmprof_unwind.h"
-#include "rvmprof_mt.h"
-
-
-/************************************************************/
-
-// functions copied from libunwind using dlopen
-
-static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
-static int (*unw_step)(unw_cursor_t*) = NULL;
-static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
-static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL;
-
-static int profile_file = -1;
-
-
-RPY_EXTERN
-char *rpython_vmprof_init(int fd)
-{
-    if (!unw_get_reg) {
-        void *libhandle;
-
-        if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL)))
-            goto error;
-        if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg")))
-            goto error;
-        if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info")))
-            goto error;
-        if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local")))
-            goto error;
-        if (!(unw_step = dlsym(libhandle, "_ULx86_64_step")))
-            goto error;
-    }
-    if (prepare_concurrent_bufs() < 0)
-        return "out of memory";
-
-    assert(fd >= 0);
-    profile_file = fd;
-    return NULL;
-
- error:
-    return dlerror();
-}
-
-/************************************************************/
-
-/* value: last bit is 1 if signals must be ignored; all other bits
-   are a counter for how many threads are currently in a signal handler */
-static long volatile signal_handler_value = 1;
-
-RPY_EXTERN
-void rpython_vmprof_ignore_signals(int ignored)
-{
-    if (!ignored) {
-        __sync_fetch_and_and(&signal_handler_value, ~1L);
-    }
-    else {
-        /* set the last bit, and wait until concurrently-running signal
-           handlers finish */
-        while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) {
-            usleep(1);
-        }
-    }
-}
-
-
-/* *************************************************************
- * functions to write a profile file compatible with gperftools
- * *************************************************************
- */
-
-#define MAX_FUNC_NAME 128
-#define MAX_STACK_DEPTH   \
-    ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
-
-#define MARKER_STACKTRACE '\x01'
-#define MARKER_VIRTUAL_IP '\x02'
-#define MARKER_TRAILER '\x03'
-
-struct prof_stacktrace_s {
-    char padding[sizeof(long) - 1];
-    char marker;
-    long count, depth;
-    void *stack[];
-};
-
-static long profile_interval_usec = 0;
-static char atfork_hook_installed = 0;
-
-
-/* ******************************************************
- * libunwind workaround for process JIT frames correctly
- * ******************************************************
- */
-
-#include "rvmprof_get_custom_offset.h"
-
-typedef struct {
-    void* _unused1;
-    void* _unused2;
-    void* sp;
-    void* ip;
-    void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4];
-} vmprof_hacked_unw_cursor_t;
-
-static int vmprof_unw_step(unw_cursor_t *cp, int first_run)
-{
-    void* ip;
-    void* sp;
-    ptrdiff_t sp_offset;
-    unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip);
-    unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
-    if (!first_run) {
-        // make sure we're pointing to the CALL and not to the first
-        // instruction after. If the callee adjusts the stack for us
-        // it's not safe to be at the instruction after
-        ip -= 1;
-    }
-    sp_offset = vmprof_unw_get_custom_offset(ip, cp);
-
-    if (sp_offset == -1) {
-        // it means that the ip is NOT in JITted code, so we can use the
-        // stardard unw_step
-        return unw_step(cp);
-    }
-    else {
-        // this is a horrible hack to manually walk the stack frame, by
-        // setting the IP and SP in the cursor
-        vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp;
-        void* bp = (void*)sp + sp_offset;
-        cp2->sp = bp;
-        bp -= sizeof(void*);
-        cp2->ip = ((void**)bp)[0];
-        // the ret is on the top of the stack minus WORD
-        return 1;
-    }
-}
-
-
-/* *************************************************************
- * functions to dump the stack trace
- * *************************************************************
- */
-
-static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext)
-{
-    void *ip;
-    int n = 0;
-    unw_cursor_t cursor;
-    unw_context_t uc = *ucontext;
-
-    int ret = unw_init_local(&cursor, &uc);
-    assert(ret >= 0);
-    (void)ret;
-
-    while (n < max_depth) {
-        if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) {
-            break;
-        }
-
-        unw_proc_info_t pip;
-        unw_get_proc_info(&cursor, &pip);
-
-        /* if n==0, it means that the signal handler interrupted us while we
-           were in the trampoline, so we are not executing (yet) the real main
-           loop function; just skip it */
-        if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) {
-            // found main loop stack frame
-            void* sp;
-            unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp);
-            void *arg_addr = (char*)sp /* + mainloop_sp_offset */;
-            void **arg_ptr = (void**)arg_addr;
-            /* if (mainloop_get_virtual_ip) {
-               ip = mainloop_get_virtual_ip(*arg_ptr);
-               } else { */
-            ip = *arg_ptr;
-        }
-
-        int first_run = (n == 0);
-        result[n++] = ip;
-        n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth);
-        if (vmprof_unw_step(&cursor, first_run) <= 0)
-            break;
-    }
-    return n;
-}
-
-
-/* *************************************************************
- * the signal handler
- * *************************************************************
- */
-
-static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
-{
-    long val = __sync_fetch_and_add(&signal_handler_value, 2L);
-
-    if ((val & 1) == 0) {
-        int saved_errno = errno;
-        int fd = profile_file;
-        assert(fd >= 0);
-
-        struct profbuf_s *p = reserve_buffer(fd);
-        if (p == NULL) {
-            /* ignore this signal: there are no free buffers right now */
-        }
-        else {
-            int depth;
-            struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data;
-            st->marker = MARKER_STACKTRACE;
-            st->count = 1;
-            st->stack[0] = GetPC((ucontext_t*)ucontext);
-            depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext);
-            depth++;  // To account for pc value in stack[0];
-            st->depth = depth;
-            p->data_offset = offsetof(struct prof_stacktrace_s, marker);
-            p->data_size = (depth * sizeof(void *) +
-                            sizeof(struct prof_stacktrace_s) -
-                            offsetof(struct prof_stacktrace_s, marker));
-            commit_buffer(fd, p);
-        }
-
-        errno = saved_errno;
-    }
-
-    __sync_sub_and_fetch(&signal_handler_value, 2L);
-}
-
-
-/* *************************************************************
- * the setup and teardown functions
- * *************************************************************
- */
-
-static int install_sigprof_handler(void)
-{
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = sigprof_handler;
-    sa.sa_flags = SA_RESTART | SA_SIGINFO;
-    if (sigemptyset(&sa.sa_mask) == -1 ||
-        sigaction(SIGPROF, &sa, NULL) == -1)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_handler(void)
-{
-    if (signal(SIGPROF, SIG_DFL) == SIG_ERR)
-        return -1;
-    return 0;
-}
-
-static int install_sigprof_timer(void)
-{
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = profile_interval_usec;
-    timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static int remove_sigprof_timer(void) {
-    static struct itimerval timer;
-    timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = 0;
-    timer.it_value.tv_sec = 0;
-    timer.it_value.tv_usec = 0;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
-        return -1;
-    return 0;
-}
-
-static void atfork_disable_timer(void) {
-    if (profile_interval_usec > 0) {
-        remove_sigprof_timer();
-    }
-}
-
-static void atfork_enable_timer(void) {
-    if (profile_interval_usec > 0) {
-        install_sigprof_timer();
-    }
-}
-
-static int install_pthread_atfork_hooks(void) {
-    /* this is needed to prevent the problems described there:
-         - http://code.google.com/p/gperftools/issues/detail?id=278
-         - http://lists.debian.org/debian-glibc/2010/03/msg00161.html
-
-        TL;DR: if the RSS of the process is large enough, the clone() syscall
-        will be interrupted by the SIGPROF before it can complete, then
-        retried, interrupted again and so on, in an endless loop.  The
-        solution is to disable the timer around the fork, and re-enable it
-        only inside the parent.
-    */
-    if (atfork_hook_installed)
-        return 0;
-    int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL);
-    if (ret != 0)
-        return -1;
-    atfork_hook_installed = 1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_enable(long interval_usec)
-{
-    assert(profile_file >= 0);
-    assert(interval_usec > 0);
-    profile_interval_usec = interval_usec;
-
-    if (install_pthread_atfork_hooks() == -1)
-        goto error;
-    if (install_sigprof_handler() == -1)
-        goto error;
-    if (install_sigprof_timer() == -1)
-        goto error;
-    rpython_vmprof_ignore_signals(0);
-    return 0;
-
- error:
-    profile_file = -1;
-    profile_interval_usec = 0;
-    return -1;
-}
-
-static int _write_all(const void *buf, size_t bufsize)
-{
-    while (bufsize > 0) {
-        ssize_t count = write(profile_file, buf, bufsize);
-        if (count <= 0)
-            return -1;   /* failed */
-        buf += count;
-        bufsize -= count;
-    }
-    return 0;
-}
-
-static int close_profile(void)
-{
-    char buf[4096];
-    ssize_t size;
-    unsigned char marker = MARKER_TRAILER;
-
-    if (_write_all(&marker, 1) < 0)
-        return -1;
-
-#ifdef __linux__
-    // copy /proc/self/maps to the end of the profile file
-    int srcfd = open("/proc/self/maps", O_RDONLY);
-    if (srcfd < 0)
-        return -1;
-
-    while ((size = read(srcfd, buf, sizeof buf)) > 0) {
-        if (_write_all(buf, size) < 0) {
-            close(srcfd);
-            return -1;
-        }
-    }
-    close(srcfd);
-#else
-    // freebsd and mac
-    sprintf(buf, "procstat -v %d", getpid());
-    FILE *srcf = popen(buf, "r");
-    if (!srcf)
-        return -1;
-
-    while ((size = fread(buf, 1, sizeof buf, src))) {
-        if (_write_all(buf, size) < 0) {
-            pclose(srcf);
-            return -1;
-        }
-    }
-    pclose(srcf);
-#endif
-
-    /* don't close() the file descriptor from here */
-    profile_file = -1;
-    return 0;
-}
-
-RPY_EXTERN
-int rpython_vmprof_disable(void)
-{
-    rpython_vmprof_ignore_signals(1);
-    profile_interval_usec = 0;
-
-    if (remove_sigprof_timer() == -1)
-        return -1;
-    if (remove_sigprof_handler() == -1)
-        return -1;
-    if (shutdown_concurrent_bufs(profile_file) < 0)
-        return -1;
-    return close_profile();
-}
-
-RPY_EXTERN
-void rpython_vmprof_write_buf(char *buf, long size)
-{
-    struct profbuf_s *p;
-
-    while ((p = reserve_buffer(profile_file)) == NULL) {
-        /* spin loop waiting for a buffer to be ready; should almost never
-           be the case */
-        usleep(1);
-    }
-
-    if (size > SINGLE_BUF_SIZE)
-        size = SINGLE_BUF_SIZE;
-    memcpy(p->data, buf, size);
-    p->data_size = size;
-
-    commit_buffer(profile_file, p);
-}
+#include "vmprof_main.h"
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -1,6 +1,6 @@
 
-RPY_EXTERN char *rpython_vmprof_init(int);
-RPY_EXTERN void rpython_vmprof_ignore_signals(int);
-RPY_EXTERN int rpython_vmprof_enable(long);
-RPY_EXTERN int rpython_vmprof_disable(void);
-RPY_EXTERN void rpython_vmprof_write_buf(char *, long);
+RPY_EXTERN char *vmprof_init(int, double, char *);
+RPY_EXTERN void vmprof_ignore_signals(int);
+RPY_EXTERN int vmprof_enable(void);
+RPY_EXTERN int vmprof_disable(void);
+RPY_EXTERN int vmprof_register_virtual_function(char *, long, int);
diff --git a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h
+++ /dev/null
@@ -1,63 +0,0 @@
-
-#ifdef PYPY_JIT_CODEMAP
-void *pypy_find_codemap_at_addr(long addr, long *start_addr);
-long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
-                                long *current_pos_addr);
-long pypy_jit_stack_depth_at_loc(long loc);
-#endif
-
-
-static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
-#ifdef PYPY_JIT_CODEMAP
-    intptr_t ip_l = (intptr_t)ip;
-    return pypy_jit_stack_depth_at_loc(ip_l);
-#else
-    return -1;
-#endif
-}
-
-static long vmprof_write_header_for_jit_addr(void **result, long n,
-                                             void *ip, int max_depth)
-{
-#ifdef PYPY_JIT_CODEMAP
-    void *codemap;
-    long current_pos = 0;
-    intptr_t id;
-    long start_addr = 0;
-    intptr_t addr = (intptr_t)ip;
-    int start, k;
-    void *tmp;
-
-    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
-    if (codemap == NULL)
-        // not a jit code at all
-        return n;
-
-    // modify the last entry to point to start address and not the random one
-    // in the middle
-    result[n - 1] = (void*)start_addr;
-    result[n] = (void*)2;
-    n++;
-    start = n;
-    while (n < max_depth) {
-        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
-        if (id == -1)
-            // finish
-            break;
-        if (id == 0)
-            continue; // not main codemap
-        result[n++] = (void *)id;
-    }
-    k = 0;
-    while (k < (n - start) / 2) {
-        tmp = result[start + k];
-        result[start + k] = result[n - k - 1];
-        result[n - k - 1] = tmp;
-        k++;
-    }
-    if (n < max_depth) {
-        result[n++] = (void*)3;
-    }
-#endif
-    return n;
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_getpc.h b/rpython/rlib/rvmprof/src/rvmprof_getpc.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rvmprof_getpc.h
+++ /dev/null
@@ -1,187 +0,0 @@
-// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright (c) 2005, Google Inc.
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are
-// met:
-//
-//     * Redistributions of source code must retain the above copyright
-// notice, this list of conditions and the following disclaimer.
-//     * Redistributions in binary form must reproduce the above
-// copyright notice, this list of conditions and the following disclaimer
-// in the documentation and/or other materials provided with the
-// distribution.
-//     * Neither the name of Google Inc. nor the names of its
-// contributors may be used to endorse or promote products derived from
-// this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-// ---
-// Author: Craig Silverstein
-//
-// This is an internal header file used by profiler.cc.  It defines
-// the single (inline) function GetPC.  GetPC is used in a signal
-// handler to figure out the instruction that was being executed when
-// the signal-handler was triggered.
-//
-// To get this, we use the ucontext_t argument to the signal-handler
-// callback, which holds the full context of what was going on when
-// the signal triggered.  How to get from a ucontext_t to a Program
-// Counter is OS-dependent.
-
-#ifndef BASE_GETPC_H_
-#define BASE_GETPC_H_
-
-#include "rvmprof_config.h"
-
-// On many linux systems, we may need _GNU_SOURCE to get access to
-// the defined constants that define the register we want to see (eg
-// REG_EIP).  Note this #define must come first!
-#define _GNU_SOURCE 1
-// If #define _GNU_SOURCE causes problems, this might work instead.
-// It will cause problems for FreeBSD though!, because it turns off
-// the needed __BSD_VISIBLE.
-//#define _XOPEN_SOURCE 500
-
-#include          // for memcmp
-#if defined(HAVE_SYS_UCONTEXT_H)
-#include 
-#elif defined(HAVE_UCONTEXT_H)
-#include        // for ucontext_t (and also mcontext_t)
-#elif defined(HAVE_CYGWIN_SIGNAL_H)
-#include 
-typedef ucontext ucontext_t;
-#endif
-
-
-// Take the example where function Foo() calls function Bar().  For
-// many architectures, Bar() is responsible for setting up and tearing
-// down its own stack frame.  In that case, it's possible for the
-// interrupt to happen when execution is in Bar(), but the stack frame
-// is not properly set up (either before it's done being set up, or
-// after it's been torn down but before Bar() returns).  In those
-// cases, the stack trace cannot see the caller function anymore.
-//
-// GetPC can try to identify this situation, on architectures where it
-// might occur, and unwind the current function call in that case to
-// avoid false edges in the profile graph (that is, edges that appear
-// to show a call skipping over a function).  To do this, we hard-code
-// in the asm instructions we might see when setting up or tearing
-// down a stack frame.
-//
-// This is difficult to get right: the instructions depend on the
-// processor, the compiler ABI, and even the optimization level.  This
-// is a best effort patch -- if we fail to detect such a situation, or
-// mess up the PC, nothing happens; the returned PC is not used for
-// any further processing.
-struct CallUnrollInfo {
-  // Offset from (e)ip register where this instruction sequence
-  // should be matched. Interpreted as bytes. Offset 0 is the next
-  // instruction to execute. Be extra careful with negative offsets in
-  // architectures of variable instruction length (like x86) - it is
-  // not that easy as taking an offset to step one instruction back!
-  int pc_offset;
-  // The actual instruction bytes. Feel free to make it larger if you
-  // need a longer sequence.
-  unsigned char ins[16];
-  // How many bytes to match from ins array?
-  int ins_size;
-  // The offset from the stack pointer (e)sp where to look for the
-  // call return address. Interpreted as bytes.
-  int return_sp_offset;
-};
-
-
-// The dereferences needed to get the PC from a struct ucontext were
-// determined at configure time, and stored in the macro
-// PC_FROM_UCONTEXT in config.h.  The only thing we need to do here,
-// then, is to do the magic call-unrolling for systems that support it.
-
-// -- Special case 1: linux x86, for which we have CallUnrollInfo
-#if defined(__linux) && defined(__i386) && defined(__GNUC__)
-static const CallUnrollInfo callunrollinfo[] = {
-  // Entry to a function:  push %ebp;  mov  %esp,%ebp
-  // Top-of-stack contains the caller IP.
-  { 0,
-    {0x55, 0x89, 0xe5}, 3,
-    0
-  },
-  // Entry to a function, second instruction:  push %ebp;  mov  %esp,%ebp
-  // Top-of-stack contains the old frame, caller IP is +4.
-  { -1,
-    {0x55, 0x89, 0xe5}, 3,
-    4
-  },
-  // Return from a function: RET.
-  // Top-of-stack contains the caller IP.
-  { 0,
-    {0xc3}, 1,
-    0
-  }
-};
-
-void* GetPC(ucontext_t *signal_ucontext) {
-  // See comment above struct CallUnrollInfo.  Only try instruction
-  // flow matching if both eip and esp looks reasonable.
-  const int eip = signal_ucontext->uc_mcontext.gregs[REG_EIP];
-  const int esp = signal_ucontext->uc_mcontext.gregs[REG_ESP];
-  if ((eip & 0xffff0000) != 0 && (~eip & 0xffff0000) != 0 &&
-      (esp & 0xffff0000) != 0) {
-    char* eip_char = reinterpret_cast(eip);
-    for (int i = 0; i < sizeof(callunrollinfo)/sizeof(*callunrollinfo); ++i) {
-      if (!memcmp(eip_char + callunrollinfo[i].pc_offset,
-                  callunrollinfo[i].ins, callunrollinfo[i].ins_size)) {
-        // We have a match.
-        void **retaddr = (void**)(esp + callunrollinfo[i].return_sp_offset);
-        return *retaddr;
-      }
-    }
-  }
-  return (void*)eip;
-}
-
-// Special case #2: Windows, which has to do something totally different.
-#elif defined(_WIN32) || defined(__CYGWIN__) || defined(__CYGWIN32__) || defined(__MINGW32__)
-// If this is ever implemented, probably the way to do it is to have
-// profiler.cc use a high-precision timer via timeSetEvent:
-//    http://msdn2.microsoft.com/en-us/library/ms712713.aspx
-// We'd use it in mode TIME_CALLBACK_FUNCTION/TIME_PERIODIC.
-// The callback function would be something like prof_handler, but
-// alas the arguments are different: no ucontext_t!  I don't know
-// how we'd get the PC (using StackWalk64?)
-//    http://msdn2.microsoft.com/en-us/library/ms680650.aspx
-
-#include "base/logging.h"   // for RAW_LOG
-#ifndef HAVE_CYGWIN_SIGNAL_H
-typedef int ucontext_t;
-#endif
-
-void* GetPC(ucontext_t *signal_ucontext) {
-  RAW_LOG(ERROR, "GetPC is not yet implemented on Windows\n");
-  return NULL;
-}
-
-// Normal cases.  If this doesn't compile, it's probably because
-// PC_FROM_UCONTEXT is the empty string.  You need to figure out
-// the right value for your system, and add it to the list in
-// configure.ac (or set it manually in your config.h).
-#else
-void* GetPC(ucontext_t *signal_ucontext) {
-  return (void*)signal_ucontext->PC_FROM_UCONTEXT;   // defined in config.h
-}
-
-#endif
-
-#endif  // BASE_GETPC_H_
diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/rvmprof_mt.h
deleted file mode 100644
--- a/rpython/rlib/rvmprof/src/rvmprof_mt.h
+++ /dev/null
@@ -1,210 +0,0 @@
-/* Support for multithreaded write() operations */
-
-#include 
-#include 
-
-/* The idea is that we have MAX_NUM_BUFFERS available, all of size
-   SINGLE_BUF_SIZE.  Threads and signal handlers can ask to reserve a
-   buffer, fill it, and finally "commit" it, at which point its
-   content is written into the profile file.  There is no hard
-   guarantee about the order in which the committed blocks are
-   actually written.  We do this with two constrains:
-
-   - write() calls should not overlap; only one thread can be
-     currently calling it.
-
-   - the code needs to be multithread-safe *and* signal-handler-safe,
-     which means it must be written in a wait-free style: never have
-     spin loops waiting for some lock to be released, from any of
-     the functions that can be called from the signal handler!  The
-     code holding the lock could be running in the same thread,
-     currently interrupted by the signal handler.
-
-   The value of MAX_NUM_BUFFERS is a trade-off between too high
-   (lots of unnecessary memory, lots of checking all of them)
-   and too low (risk that there is none left).
-*/
-#define MAX_NUM_BUFFERS  20
-#define SINGLE_BUF_SIZE  (8192 - 2 * sizeof(unsigned int))
-
-#if defined(__i386__) || defined(__amd64__)
-  static inline void write_fence(void) { asm("" : : : "memory"); }
-#else
-  static inline void write_fence(void) { __sync_synchronize(); }
-#endif
-
-
-#define PROFBUF_UNUSED   0
-#define PROFBUF_FILLING  1
-#define PROFBUF_READY    2
-
-
-struct profbuf_s {
-    unsigned int data_size;
-    unsigned int data_offset;
-    char data[SINGLE_BUF_SIZE];
-};
-
-static char volatile profbuf_state[MAX_NUM_BUFFERS];
-static struct profbuf_s *profbuf_all_buffers = NULL;
-static int volatile profbuf_write_lock = 2;
-static long profbuf_pending_write;
-
-
-static void unprepare_concurrent_bufs(void)
-{
-    if (profbuf_all_buffers != NULL) {
-        munmap(profbuf_all_buffers, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS);
-        profbuf_all_buffers = NULL;
-    }
-}
-
-static int prepare_concurrent_bufs(void)
-{
-    assert(sizeof(struct profbuf_s) == 8192);
-
-    unprepare_concurrent_bufs();
-    profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS,
-                               PROT_READ | PROT_WRITE,
-                               MAP_PRIVATE | MAP_ANONYMOUS,
-                               -1, 0);
-    if (profbuf_all_buffers == MAP_FAILED) {
-        profbuf_all_buffers = NULL;
-        return -1;
-    }
-    memset((char *)profbuf_state, PROFBUF_UNUSED, sizeof(profbuf_state));
-    profbuf_write_lock = 0;
-    profbuf_pending_write = -1;
-    return 0;
-}
-
-static int _write_single_ready_buffer(int fd, long i)
-{
-    /* Try to write to disk the buffer number 'i'.  This function must
-       only be called while we hold the write lock. */
-    assert(profbuf_write_lock != 0);
-
-    if (profbuf_pending_write >= 0) {
-        /* A partially written buffer is waiting.  We'll write the
-           rest of this buffer now, instead of 'i'. */
-        i = profbuf_pending_write;
-        assert(profbuf_state[i] == PROFBUF_READY);
-    }
-
-    if (profbuf_state[i] != PROFBUF_READY) {
-        /* this used to be a race condition: the buffer was written by a
-           different thread already, nothing to do now */
-        return 0;
-    }
-
-    int err;
-    struct profbuf_s *p = &profbuf_all_buffers[i];
-    ssize_t count = write(fd, p->data + p->data_offset, p->data_size);
-    if (count == p->data_size) {
-        profbuf_state[i] = PROFBUF_UNUSED;
-        profbuf_pending_write = -1;
-    }
-    else {
-        if (count > 0) {
-            p->data_offset += count;
-            p->data_size -= count;
-        }
-        profbuf_pending_write = i;
-        if (count < 0)
-            return -1;
-    }
-    return 0;
-}
-
-static void _write_ready_buffers(int fd)
-{
-    long i;
-    int has_write_lock = 0;
-
-    for (i = 0; i < MAX_NUM_BUFFERS; i++) {
-        if (profbuf_state[i] == PROFBUF_READY) {
-            if (!has_write_lock) {
-                if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 1))
-                    return;   /* can't acquire the write lock, give up */
-                has_write_lock = 1;
-            }
-            if (_write_single_ready_buffer(fd, i) < 0)
-                break;
-        }
-    }
-    if (has_write_lock)
-        profbuf_write_lock = 0;
-}
-
-static struct profbuf_s *reserve_buffer(int fd)
-{
-    /* Tries to enter a region of code that fills one buffer.  If
-       successful, returns the profbuf_s.  It fails only if the
-       concurrent buffers are all busy (extreme multithreaded usage).
-
-       This might call write() to emit the data sitting in
-       previously-prepared buffers.  In case of write() error, the
-       error is ignored but unwritten data stays in the buffers.
-    */
-    long i;
-
-    _write_ready_buffers(fd);
-
-    for (i = 0; i < MAX_NUM_BUFFERS; i++) {
-        if (profbuf_state[i] == PROFBUF_UNUSED &&
-            __sync_bool_compare_and_swap(&profbuf_state[i], PROFBUF_UNUSED,
-                                         PROFBUF_FILLING)) {
-            struct profbuf_s *p = &profbuf_all_buffers[i];
-            p->data_size = 0;
-            p->data_offset = 0;
-            return p;
-        }
-    }
-    /* no unused buffer found */
-    return NULL;
-}
-
-static void commit_buffer(int fd, struct profbuf_s *buf)
-{
-    /* Leaves a region of code that filled 'buf'.
-
-       This might call write() to emit the data now ready.  In case of
-       write() error, the error is ignored but unwritten data stays in
-       the buffers.
-    */
-
-    /* Make sure every thread sees the full content of 'buf' */
-    write_fence();
-
-    /* Then set the 'ready' flag */
-    long i = buf - profbuf_all_buffers;
-    assert(profbuf_state[i] == PROFBUF_FILLING);
-    profbuf_state[i] = PROFBUF_READY;
-
-    if (!__sync_bool_compare_and_swap(&profbuf_write_lock, 0, 1)) {
-        /* can't acquire the write lock, ignore */
-    }
-    else {
-        _write_single_ready_buffer(fd, i);
-        profbuf_write_lock = 0;
-    }
-}
-
-static int shutdown_concurrent_bufs(int fd)
-{
-    /* no signal handler can be running concurrently here, because we
-       already did rpython_vmprof_ignore_signals(1) */
-    assert(profbuf_write_lock == 0);
-    profbuf_write_lock = 2;
-
-    /* last attempt to flush buffers */
-    int i;
-    for (i = 0; i < MAX_NUM_BUFFERS; i++) {
-        while (profbuf_state[i] == PROFBUF_READY) {
-            if (_write_single_ready_buffer(fd, i) < 0)
-                return -1;
-        }
-    }
-    unprepare_concurrent_bufs();
-    return 0;
-}
diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h
rename from rpython/rlib/rvmprof/src/rvmprof_config.h
rename to rpython/rlib/rvmprof/src/vmprof_config.h
diff --git a/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h
@@ -0,0 +1,120 @@
+
+#ifdef PYPY_JIT_CODEMAP
+void *pypy_find_codemap_at_addr(long addr, long *start_addr);
+long pypy_yield_codemap_at_addr(void *codemap_raw, long addr,
+                                long *current_pos_addr);
+long pypy_jit_stack_depth_at_loc(long loc);
+#endif
+
+
+#ifdef CPYTHON_GET_CUSTOM_OFFSET
+static void *tramp_start, *tramp_end;
+#endif
+
+
+static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) {
+
+#if defined(PYPY_JIT_CODEMAP)
+
+    intptr_t ip_l = (intptr_t)ip;
+    return pypy_jit_stack_depth_at_loc(ip_l);
+
+#elif defined(CPYTHON_GET_CUSTOM_OFFSET)
+
+    if (ip >= tramp_start && ip <= tramp_end) {
+        // XXX the return value is wrong for all the places before push and
+        //     after pop, fix
+        void *bp;
+        void *sp;
+
+        /* This is a stage2 trampoline created by hotpatch:
+
+               push   %rbx
+               push   %rbp
+               mov    %rsp,%rbp
+               and    $0xfffffffffffffff0,%rsp   // make sure the stack is aligned
+               movabs $0x7ffff687bb10,%rbx
+               callq  *%rbx
+               leaveq 
+               pop    %rbx
+               retq   
+
+           the stack layout is like this:
+
+               +-----------+                      high addresses
+               | ret addr  |
+               +-----------+
+               | saved rbx |   start of the function frame
+               +-----------+
+               | saved rbp |
+               +-----------+
+               | ........  |   <-- rbp
+               +-----------+                      low addresses
+
+           So, the trampoline frame starts at rbp+16, and the return address,
+           is at rbp+24.  The vmprof API requires us to return the offset of
+           the frame relative to sp, hence we have this weird computation.
+
+           XXX (antocuni): I think we could change the API to return directly
+           the frame address instead of the offset; however, this require a
+           change in the PyPy code too
+        */
+
+        unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp);
+        unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp);
+        return bp+16+8-sp;
+    }
+    return -1;
+
+#else
+
+    return -1;
+
+#endif
+}
+
+static long vmprof_write_header_for_jit_addr(void **result, long n,
+                                             void *ip, int max_depth)
+{
+#ifdef PYPY_JIT_CODEMAP
+    void *codemap;
+    long current_pos = 0;
+    intptr_t id;
+    long start_addr = 0;
+    intptr_t addr = (intptr_t)ip;
+    int start, k;
+    void *tmp;
+
+    codemap = pypy_find_codemap_at_addr(addr, &start_addr);
+    if (codemap == NULL)
+        // not a jit code at all
+        return n;
+
+    // modify the last entry to point to start address and not the random one
+    // in the middle
+    result[n - 1] = (void*)start_addr;
+    result[n] = (void*)2;
+    n++;
+    start = n;
+    while (n < max_depth) {
+        id = pypy_yield_codemap_at_addr(codemap, addr, ¤t_pos);
+        if (id == -1)
+            // finish
+            break;
+        if (id == 0)
+            continue; // not main codemap
+        result[n++] = (void *)id;
+    }
+    k = 0;
+    while (k < (n - start) / 2) {
+        tmp = result[start + k];
+        result[start + k] = result[n - k - 1];
+        result[n - k - 1] = tmp;
+        k++;
+    }
+    if (n < max_depth) {
+        result[n++] = (void*)3;
+    }
+#endif
+    return n;
+}
diff --git a/rpython/rlib/rvmprof/src/vmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h
@@ -0,0 +1,193 @@
+// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
+// Copyright (c) 2005, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

From noreply at buildbot.pypy.org  Tue Aug 25 23:41:56 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Tue, 25 Aug 2015 23:41:56 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Add more tests for zip-importing an
 archive whose filename contain non-ASCII characters, and fix bugs.
Message-ID: <20150825214156.9DA561C088E@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79230:6a89b5539f61
Date: 2015-08-25 23:41 +0200
http://bitbucket.org/pypy/pypy/changeset/6a89b5539f61/

Log:	Add more tests for zip-importing an archive whose filename contain
	non-ASCII characters, and fix bugs.

diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py
--- a/pypy/module/imp/importing.py
+++ b/pypy/module/imp/importing.py
@@ -350,7 +350,7 @@
     start = len(filename) - 4
     stop = len(filename) - 1
     if not 0 <= start <= stop or filename[start:stop].lower() != ".py":
-        return space.wrap(filename)
+        return space.wrap_fsdecoded(filename)
     py = make_source_pathname(filename)
     if py is None:
         py = filename[:-1]
@@ -360,8 +360,8 @@
         pass
     else:
         if stat.S_ISREG(st.st_mode):
-            return space.wrap(py)
-    return space.wrap(filename)
+            return space.wrap_fsdecoded(py)
+    return space.wrap_fsdecoded(filename)
 
 def update_code_filenames(space, code_w, pathname, oldname=None):
     assert isinstance(code_w, PyCode)
diff --git a/pypy/module/zipimport/interp_zipimport.py b/pypy/module/zipimport/interp_zipimport.py
--- a/pypy/module/zipimport/interp_zipimport.py
+++ b/pypy/module/zipimport/interp_zipimport.py
@@ -146,7 +146,7 @@
 
     def import_py_file(self, space, modname, filename, buf, pkgpath):
         w = space.wrap
-        w_mod = w(Module(space, w(modname)))
+        w_mod = w(Module(space, space.wrap_fsdecoded(modname)))
         real_name = self.filename + os.path.sep + self.corr_zname(filename)
         space.setattr(w_mod, w('__loader__'), space.wrap(self))
         importing._prepare_module(space, w_mod, real_name, pkgpath)
@@ -313,8 +313,8 @@
                         space, co_filename, source)
                 return space.wrap(code_w)
         raise oefmt(get_error(space),
-                    "Cannot find source or code for %s in %s",
-                    filename, self.name)
+                    "Cannot find source or code for %s in %R",
+                    filename, space.wrap_fsdecoded(self.name))
 
     @unwrap_spec(fullname='str0')
     def get_source(self, space, fullname):
@@ -334,17 +334,19 @@
             # We have the module, but no source.
             return space.w_None
         raise oefmt(get_error(space),
-                    "Cannot find source for %s in %s", filename, self.name)
+                    "Cannot find source for %s in %R", filename,
+                    space.wrap_fsdecoded(self.name))
 
     @unwrap_spec(fullname='str0')
     def get_filename(self, space, fullname):
         filename = self.make_filename(fullname)
         for _, is_package, ext in ENUMERATE_EXTS:
             if self.have_modulefile(space, filename + ext):
-                return space.wrap(self.filename + os.path.sep +
-                                  self.corr_zname(filename + ext))
+                return space.wrap_fsdecoded(self.filename + os.path.sep +
+                                            self.corr_zname(filename + ext))
         raise oefmt(get_error(space),
-                    "Cannot find module %s in %s", filename, self.name)
+                    "Cannot find module %s in %R", filename,
+                    space.wrap_fsdecoded(self.name))
 
     @unwrap_spec(fullname='str0')
     def is_package(self, space, fullname):
@@ -353,11 +355,12 @@
             if self.have_modulefile(space, filename + ext):
                 return space.wrap(is_package)
         raise oefmt(get_error(space),
-                    "Cannot find module %s in %s", filename, self.name)
+                    "Cannot find module %s in %R", filename,
+                    space.wrap_fsdecoded(self.name))
 
     def getarchive(self, space):
         space = self.space
-        return space.wrap(self.filename)
+        return space.wrap_fsdecoded(self.filename)
 
 def descr_new_zipimporter(space, w_type, w_name):
     name = space.fsencode_w(w_name)
@@ -378,8 +381,8 @@
             ok = True
             break
     if not ok:
-        raise oefmt(get_error(space), "Did not find %s to be a valid zippath",
-                    name)
+        raise oefmt(get_error(space), "Did not find %R to be a valid zippath",
+                    w_name)
     try:
         w_result = zip_cache.get(filename)
         if w_result is None:
@@ -391,7 +394,8 @@
     try:
         zip_file = RZipFile(filename, 'r')
     except (BadZipfile, OSError):
-        raise oefmt(get_error(space), "%s seems not to be a zipfile", filename)
+        raise oefmt(get_error(space), "%R seems not to be a zipfile",
+                    space.wrap_fsdecoded(filename))
     except RZlibError, e:
         # in this case, CPython raises the direct exception coming
         # from the zlib module: let's to the same
diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py
--- a/pypy/module/zipimport/test/test_zipimport.py
+++ b/pypy/module/zipimport/test/test_zipimport.py
@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 import inspect
 import os
 import time
@@ -346,11 +347,48 @@
         import zipimport
         assert sys.path_hooks.count(zipimport.zipimporter) == 1
 
-    def test_unicode_filename(self):
+    def w__make_unicode_filename(self):
+        import os
+        head, tail = os.path.split(self.zipfile)
+        self.zipfile = head + os.path.sep + tail[:4] + '_ä' + tail[4:]
+
+    def test_unicode_filename_notfound(self):
         import zipimport
-        raises(zipimport.ZipImportError, 
+        raises(zipimport.ZipImportError,
                zipimport.zipimporter, 'caf\xe9')
 
+    def test_unicode_filename_invalid_zippath(self):
+        import zipimport
+        import os
+        self._make_unicode_filename()
+        os.mkdir(self.zipfile)
+        raises(zipimport.ZipImportError,
+               zipimport.zipimporter, self.zipfile)
+
+    def test_unicode_filename_invalid_zip(self):
+        import zipimport
+        self._make_unicode_filename()
+        open(self.zipfile, 'wb').write(b'invalid zip')
+        raises(zipimport.ZipImportError,
+               zipimport.zipimporter, self.zipfile)
+
+    def test_unicode_filename_existing(self):
+        import zipimport
+        self._make_unicode_filename()
+        self.writefile('ä.py', '3')
+        z = zipimport.zipimporter(self.zipfile)
+        assert not z.is_package('ä')
+        raises(ImportError, z.is_package, 'xx')
+        assert z.get_source('ä') == '3'
+        raises(ImportError, z.get_source, 'xx')
+        assert z.get_code('ä')
+        raises(ImportError, z.get_code, 'xx')
+        mod = z.load_module('ä')
+        assert z.get_filename('ä') == mod.__file__
+        raises(ImportError, z.load_module, 'xx')
+        raises(ImportError, z.get_filename, 'xx')
+        assert z.archive == self.zipfile
+
     def test_co_filename(self):
         self.writefile('mymodule.py', """
 def get_co_filename():

From noreply at buildbot.pypy.org  Wed Aug 26 09:05:43 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Aug 2015 09:05:43 +0200 (CEST)
Subject: [pypy-commit] cffi default: Issue #217: use memcpy() instead of
 manually copying the data. On
Message-ID: <20150826070543.9032B1C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2256:b8169a197564
Date: 2015-08-26 09:04 +0200
http://bitbucket.org/cffi/cffi/changeset/b8169a197564/

Log:	Issue #217: use memcpy() instead of manually copying the data. On
	some architecture, the target is not correctly aligned for "void *".

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -3582,7 +3582,6 @@
            array      [ctype, length]
            funcptr    [ctresult, ellipsis+abi, num_args, ctargs...]
     */
-    long i;
     PyObject *key, *y;
     const void **pkey;
     int err;
@@ -3592,8 +3591,7 @@
         goto error;
 
     pkey = (const void **)PyBytes_AS_STRING(key);
-    for (i = 0; i < keylength; i++)
-        pkey[i] = unique_key[i];
+    memcpy(pkey, unique_key, keylength * sizeof(void *));
 
     y = PyDict_GetItem(unique_cache, key);
     if (y != NULL) {

From noreply at buildbot.pypy.org  Wed Aug 26 09:08:56 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 26 Aug 2015 09:08:56 +0200 (CEST)
Subject: [pypy-commit] cffi default: Document issue #217
Message-ID: <20150826070856.B9DD01C0473@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2257:c1df0e533840
Date: 2015-08-26 09:09 +0200
http://bitbucket.org/cffi/cffi/changeset/c1df0e533840/

Log:	Document issue #217

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,13 @@
 ======================
 
 
+v1.2.2
+======
+
+* Issue #217: fix possible unaligned pointer manipulation, which crash
+  on some architectures (64-bit, non-x86).
+
+
 v1.2.1
 ======
 

From noreply at buildbot.pypy.org  Wed Aug 26 16:39:23 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 26 Aug 2015 16:39:23 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: questions with failing test about
	finalizers
Message-ID: <20150826143923.B42A51C0473@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1945:824bc70d6f36
Date: 2015-08-26 16:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/824bc70d6f36/

Log:	questions with failing test about finalizers

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -481,6 +481,13 @@
     LIST_FREE(f->run_finalizers);
 }
 
+/* XXX: can there be a race between _invoke_general_finalizers
+        and _commit_finalizer on g_finalizers (+other places?)?
+   XXX: what happens in _execute_finalizer if the transaction
+        conflicts (or fails to become inevitable) in a finalizer?
+        (the run_finalizers list is half-way cleared?)
+*/
+
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
     /* called between transactions */
@@ -496,6 +503,7 @@
     stm_rewind_jmp_enterframe(tl, &rjbuf);
     _stm_start_transaction(tl);
 
+    fprintf(stderr, "run_finalizers: %lu\n", list_count(g_finalizers.run_finalizers));
     _execute_finalizers(&g_finalizers);
 
     _stm_commit_transaction();
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -139,6 +139,7 @@
         self.expect_finalized([lp1], from_tlnum=0)
 
 
+
 class TestRegularFinalizer(BaseTest):
     expect_content_character = None
     run_major_collect_in_finalizer = False
@@ -272,3 +273,59 @@
         self.expect_finalized([])
         stm_major_collect()
         self.expect_finalized([lp1])
+
+
+class TestMoreRegularFinalizers(BaseTest):
+
+    def test_inevitable_in_finalizer(self):
+        lpo = stm_allocate_old(16)
+
+        self._first_time = True
+        @ffi.callback("void(object_t *)")
+        def finalizer(obj):
+            print "finalizing!", obj
+            stm_set_char(lpo, 'a')
+
+            if self._first_time:
+                self._first_time = False
+                # we will switch to the other TX and
+                # make it inevitable, so that our TX
+                # will abort on commit (or validate)
+                self.switch(0, validate=False)
+                self.become_inevitable()
+                self.switch(1, validate=False)
+
+        lib.stmcb_finalizer = finalizer
+        self._finalizer_keepalive = finalizer
+
+        # start a transaction with a finalizing obj
+        self.switch(1)
+        self.start_transaction()
+        lpf = stm_allocate_with_finalizer(16)
+
+        self.push_root(lpf)
+        stm_minor_collect()
+
+
+        self.switch(0)
+        self.start_transaction()
+        stm_set_char(lpo, 'x')
+        self.switch(1)
+        lpf = self.pop_root()
+        # commit this TX, start a new one, let lpf
+        # die with a major-gc:
+        self.commit_transaction()
+        self.start_transaction()
+        stm_major_collect()
+        # commit and run finalizer in separate TX
+        # that will abort because of a conflict
+        self.commit_transaction()
+
+        self.switch(0, validate=False)
+        assert stm_get_char(lpo) == 'x'
+        # commit the now-inevitable TX and run
+        # the aborted finalizer again
+        self.commit_transaction()
+        self.start_transaction()
+        # should now see the value set by finalizer
+        assert stm_get_char(lpo) == 'a'

From noreply at buildbot.pypy.org  Wed Aug 26 17:18:45 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Wed, 26 Aug 2015 17:18:45 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: add another question about light
	finalizers
Message-ID: <20150826151845.4BAFA1C0557@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1946:9076b244ab08
Date: 2015-08-26 17:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/9076b244ab08/

Log:	add another question about light finalizers

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -486,6 +486,10 @@
    XXX: what happens in _execute_finalizer if the transaction
         conflicts (or fails to become inevitable) in a finalizer?
         (the run_finalizers list is half-way cleared?)
+   XXX: according to translator.backendopt.finalizer, getfield_gc
+        for primitive types is a safe op in light finalizers.
+        I don't think that's correct in general (maybe if
+        getfield on *dying obj*).
 */
 
 static void _invoke_general_finalizers(stm_thread_local_t *tl)

From noreply at buildbot.pypy.org  Wed Aug 26 18:06:50 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 26 Aug 2015 18:06:50 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: Fix an error at interpreter shutdown in
 case stdout or stderr is missing.
Message-ID: <20150826160650.46E751C0684@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3k
Changeset: r79231:07bfad12ceb6
Date: 2015-08-26 01:57 +0200
http://bitbucket.org/pypy/pypy/changeset/07bfad12ceb6/

Log:	Fix an error at interpreter shutdown in case stdout or stderr is
	missing.

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -130,8 +130,8 @@
                """)
 
     def flush_std_files(self, space):
-        w_stdout = space.sys.get('stdout')
-        w_stderr = space.sys.get('stderr')
+        w_stdout = space.sys.getdictvalue(space, 'stdout')
+        w_stderr = space.sys.getdictvalue(space, 'stderr')
         for w_file in [w_stdout, w_stderr]:
             if not (space.is_none(w_file) or
                     self._file_is_closed(space, w_file)):
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -37,6 +37,18 @@
         space.setattr(w_sys, space.wrap('stdout'), w_sys.get('__stdout__'))
         space.setattr(w_sys, space.wrap('stderr'), w_sys.get('__stderr__'))
 
+def test_stdio_missing_at_shutdown(space):
+    w_sys = space.sys
+
+    try:
+        space.delattr(w_sys, space.wrap('stdout'))
+        space.delattr(w_sys, space.wrap('stderr'))
+        # called at shtudown
+        w_sys.flush_std_files(space)
+    finally:
+        space.setattr(w_sys, space.wrap('stdout'), w_sys.get('__stdout__'))
+        space.setattr(w_sys, space.wrap('stderr'), w_sys.get('__stderr__'))
+
 class AppTestAppSysTests:
 
     def setup_class(cls):

From noreply at buildbot.pypy.org  Wed Aug 26 18:06:52 2015
From: noreply at buildbot.pypy.org (mjacob)
Date: Wed, 26 Aug 2015 18:06:52 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Implement preliminary stderr printer and
 use it in app_main.py.
Message-ID: <20150826160652.70A081C0684@cobra.cs.uni-duesseldorf.de>

Author: Manuel Jacob 
Branch: py3.3
Changeset: r79232:54b7b89a7172
Date: 2015-08-26 18:06 +0200
http://bitbucket.org/pypy/pypy/changeset/54b7b89a7172/

Log:	Implement preliminary stderr printer and use it in app_main.py.

	This is needed because during initialization of stderr imports can
	happen. If the interpreter is run in verbose mode importlib prints
	to stderr, resulting in an AttributeError.

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -43,9 +43,10 @@
 """
 
 try:
-    from __pypy__ import hidden_applevel
+    from __pypy__ import hidden_applevel, StdErrPrinter
 except ImportError:
     hidden_applevel = lambda f: f
+    StdErrPrinter = None
 try:
     from _ast import PyCF_ACCEPT_NULL_BYTES
 except ImportError:
@@ -266,6 +267,9 @@
     if hasattr(sys, 'stdin'):
         return # already initialized
 
+    if StdErrPrinter is not None:
+        sys.stderr = sys.__stderr__ = StdErrPrinter(2)
+
     if 1:  # keep indentation
         if encoding and ':' in encoding:
             encoding, errors = encoding.split(':', 1)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -87,6 +87,7 @@
         'save_module_content_for_future_reload':
                           'interp_magic.save_module_content_for_future_reload',
         'normalize_exc'             : 'interp_magic.normalize_exc',
+        'StdErrPrinter'             : 'interp_stderrprinter.W_StdErrPrinter',
     }
 
     submodules = {
diff --git a/pypy/module/__pypy__/interp_stderrprinter.py b/pypy/module/__pypy__/interp_stderrprinter.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/interp_stderrprinter.py
@@ -0,0 +1,70 @@
+import errno, os
+
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.error import wrap_oserror
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.typedef import GetSetProperty, TypeDef
+
+
+class W_StdErrPrinter(W_Root):
+    @staticmethod
+    @unwrap_spec(fd='c_int')
+    def descr_new(space, w_subtype, fd):
+        return W_StdErrPrinter(fd)
+
+    def __init__(self, fd):
+        self.fd = fd
+
+    def descr_repr(self, space):
+        addrstring = unicode(self.getaddrstring(space))
+        return space.wrap(u"" %
+                          (self.fd, addrstring))
+
+    def descr_noop(self, space):
+        pass
+
+    def descr_fileno(self, space):
+        return space.wrap(self.fd)
+
+    def descr_isatty(self, space):
+        try:
+            res = os.isatty(self.fd)
+        except OSError, e:
+            raise wrap_oserror(space, e)
+        return space.wrap(res)
+
+    def descr_write(self, space, w_data):
+        # Encode to UTF-8.
+        data = space.identifier_w(w_data)
+
+        try:
+            n = os.write(self.fd, data)
+        except OSError, e:
+            if e.errno == errno.EAGAIN:
+                return space.w_None
+            raise wrap_oserror(space, e)
+        return space.wrap(n)
+
+    def descr_get_closed(self, space):
+        return space.wrap(False)
+
+    def descr_get_encoding(self, space):
+        return space.w_None
+
+    def descr_get_mode(self, space):
+        return space.wrap(u'w')
+
+
+W_StdErrPrinter.typedef = TypeDef("StdErrPrinter",
+    __new__ = interp2app(W_StdErrPrinter.descr_new),
+    __repr__ = interp2app(W_StdErrPrinter.descr_repr),
+    close = interp2app(W_StdErrPrinter.descr_noop),
+    flush = interp2app(W_StdErrPrinter.descr_noop),
+    fileno = interp2app(W_StdErrPrinter.descr_fileno),
+    isatty = interp2app(W_StdErrPrinter.descr_isatty),
+    write = interp2app(W_StdErrPrinter.descr_write),
+
+    closed = GetSetProperty(W_StdErrPrinter.descr_get_closed),
+    encoding = GetSetProperty(W_StdErrPrinter.descr_get_encoding),
+    mode = GetSetProperty(W_StdErrPrinter.descr_get_mode),
+)
diff --git a/pypy/module/__pypy__/test/test_stderrprinter.py b/pypy/module/__pypy__/test/test_stderrprinter.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__pypy__/test/test_stderrprinter.py
@@ -0,0 +1,16 @@
+def app_test_stderrprinter():
+    from __pypy__ import StdErrPrinter
+
+    p = StdErrPrinter(2)
+    assert repr(p).startswith("

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79233:93de8af48b53
Date: 2015-08-26 22:24 +0200
http://bitbucket.org/pypy/pypy/changeset/93de8af48b53/

Log:	rework the tests until most of them passes. Skip one of the tests
	where it's not really relevant getfield vs setfield. Emit a few more
	guards in the short preamble, I think it's ok, they're repeating
	guards from virtual state so they get optimized away

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -67,7 +67,7 @@
         pass
 
     def make_guards(self, op, short):
-        pass
+        short.append(ResOperation(rop.GUARD_VALUE, [op, self._const]))
     
 class NonNullPtrInfo(PtrInfo):
     _attrs_ = ('last_guard_pos',)
@@ -698,3 +698,9 @@
                                         lgt, mode)
 
     
+class FloatConstInfo(AbstractInfo):
+    def __init__(self, const):
+        self._const = const
+
+    def make_guards(self, op, short):
+        short.append(ResOperation(rop.GUARD_VALUE, [op, self._const]))
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -417,21 +417,22 @@
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_string)
         array = self.getptrinfo(op.getarg(0))
-        new_op = self.get_box_replacement(op)
-        if not new_op.is_constant():
-            new_op.set_forwarded(array.getlenbound(vstring.mode_string))
+        self.optimizer.setintbound(op, array.getlenbound(vstring.mode_string))
 
     def optimize_UNICODELEN(self, op):
         self.emit_operation(op)
         self.make_nonnull_str(op.getarg(0), vstring.mode_unicode)
         array = self.getptrinfo(op.getarg(0))
-        new_op = self.get_box_replacement(op)
-        if not new_op.is_constant():
-            new_op.set_forwarded(array.getlenbound(vstring.mode_unicode))
+        self.optimizer.setintbound(op, array.getlenbound(vstring.mode_unicode))
 
     def optimize_STRGETITEM(self, op):
         self.emit_operation(op)
         v1 = self.getintbound(op)
+        v2 = self.getptrinfo(op.getarg(0))
+        intbound = self.getintbound(op.getarg(1))
+        if (intbound.has_lower and v2 is not None and
+            v2.getlenbound(v2.mode) is not None):
+            v2.getlenbound(v2.mode).make_ge(IntLowerBound(intbound.lower + 1))
         v1.make_ge(IntLowerBound(0))
         v1.make_lt(IntUpperBound(256))
 
@@ -470,6 +471,11 @@
         self.emit_operation(op)
         b1 = self.getintbound(op)
         b1.make_ge(IntLowerBound(0))
+        v2 = self.getptrinfo(op.getarg(0))
+        intbound = self.getintbound(op.getarg(1))
+        if (intbound.has_lower and v2 is not None and
+            v2.getlenbound(v2.mode) is not None):
+            v2.getlenbound(v2.mode).make_ge(IntLowerBound(intbound.lower + 1))
 
     def make_int_lt(self, box1, box2):
         b1 = self.getintbound(box1)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -295,6 +295,15 @@
         for o in self.optimizations:
             o.force_at_end_of_preamble()
 
+    def force_box_for_end_of_preamble(self, box):
+        if box.type == 'r':
+            info = self.getptrinfo(box)
+            if info is not None and info.is_virtual():
+                rec = {}
+                return info.force_at_the_end_of_preamble(box, self, rec)
+            return box
+        return box
+
     def flush(self):
         for o in self.optimizations:
             o.flush()
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -388,10 +388,10 @@
             self.short.append(ResOperation(rop.GUARD_NO_OVERFLOW, [], None))
         info = preamble_op.get_forwarded()
         preamble_op.set_forwarded(None)
+        if optimizer is not None:
+            optimizer.setinfo_from_preamble(box, info, None)
         if info is not empty_info:
             info.make_guards(preamble_op, self.short)
-        if optimizer is not None:
-            optimizer.setinfo_from_preamble(box, info, None)
         return preamble_op
 
     def add_preamble_op(self, preamble_op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4892,7 +4892,7 @@
     def test_setgetfield_raw(self):
         ops = """
         [p4, p7, i30]
-        p16 = getfield_gc_r(p4, descr=valuedescr)
+        p16 = getfield_gc_r(p4, descr=nextdescr)
         guard_value(p16, ConstPtr(myptr), descr=) []
         i1 = getfield_raw_i(p7, descr=nextdescr)
         i2 = int_add(i1, i30)
@@ -4913,7 +4913,7 @@
     def test_setgetarrayitem_raw(self):
         ops = """
         [p4, i7, i30]
-        p16 = getfield_gc_r(p4, descr=valuedescr)
+        p16 = getfield_gc_r(p4, descr=otherdescr)
         guard_value(p16, ConstPtr(myptr), descr=) []
         i1 = getarrayitem_raw_i(i7, 1, descr=arraydescr)
         i2 = int_add(i1, i30)
@@ -6379,8 +6379,8 @@
     def test_str_concat_consts(self):
         ops = """
         []
-        p1 = same_as(s"ab")
-        p2 = same_as(s"cde")
+        p1 = same_as_r(s"ab")
+        p2 = same_as_r(s"cde")
         p3 = call_r(0, p1, p2, descr=strconcatdescr)
         escape_n(p3)
         jump()
@@ -6404,13 +6404,12 @@
         [p1, i1, i2, i3]
         escape_n(i3)
         i4 = int_sub(i2, i1)
-        i5 = same_as_i(i4)
-        jump(p1, i1, i2, i4, i5)
-        """
-        expected = """
-        [p1, i1, i2, i3, i4]
+        jump(p1, i1, i2, i4)
+        """
+        expected = """
+        [p1, i1, i2, i3]
         escape_n(i3)
-        jump(p1, i1, i2, i4, i4)
+        jump(p1, i1, i2, i3)
         """
         self.optimize_strunicode_loop(ops, expected, preamble)
 
@@ -7249,21 +7248,26 @@
         """
         short = """
         [p1, p187, i184]
+        guard_nonnull(p187) []
+        i10 = arraylen_gc(p187, descr=gcarraydescr)
+        i11 = int_ge(i10, 43)
+        guard_true(i11) []
         p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr)
-        #guard_value(p188, ConstPtr(myptr)) []
+        guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         jump(p25)
         """
         expected = """
         [p25, p187, i184, p189]
-        jump(p25, p187, i184, p189) # <- XXX is this correct for bridges?
+        ifoo = arraylen_gc(p187, descr=gcarraydescr) # killed by backend
+        jump(p189, p187, i184, p189)
         """
         self.optimize_loop(ops, expected, preamble, expected_short=short)
 
     def test_constant_getfield1bis(self):
         ops = """
         [p1, p187, i184]
-        p188 = getarrayitem_gc_r(p187, 42, descr=arraydescr)
+        p188 = getarrayitem_gc_r(p187, 42, descr=gcarraydescr)
         guard_value(p188, ConstPtr(myptr)) []
         p25 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         p26 = call_r(p25, descr=nonwritedescr)
@@ -7272,6 +7276,7 @@
         expected = """
         [p24, p187, i184, p25]
         p26 = call_r(p25, descr=nonwritedescr)
+        ifoo = arraylen_gc(p187, descr=gcarraydescr) # killed
         jump(p26, p187, i184, p25)
         """
         self.optimize_loop(ops, expected)
@@ -7431,6 +7436,7 @@
         """
         short = """
         [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
         p1 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull_class(p1, ConstClass(node_vtable)) []
         p2 = getfield_gc_r(p1, descr=nextdescr)
@@ -7531,10 +7537,11 @@
         """
         short = """
         [p9]
+        guard_nonnull(p9) []
         i843 = strlen(p9)
         i848 = int_ge(i843, 0)
         guard_true(i848)[]
-        jump(p9, i843)
+        jump(i843)
         """
         expected = """
         [p9, i2]
@@ -7625,6 +7632,7 @@
         """
         short = """
         [p0]
+        guard_nonnull(p0) []
         p1 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p1) []
         i1 = arraylen_gc(p1, descr=gcarraydescr)
@@ -7728,6 +7736,7 @@
         expected = """
         [p9, i1, i843]
         call_n(i843, descr=nonwritedescr)
+        ifoo = arraylen_gc(p9)
         jump(p9, i1, i843)
         """
         self.optimize_loop(ops, expected)
@@ -7742,6 +7751,7 @@
         """
         short = """
         [p0]
+        guard_nonnull(p0) []
         p1 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p1) []
         i1 = arraylen_gc(p1)
@@ -7768,6 +7778,7 @@
         """
         short = """
         [p0]
+        guard_nonnull(p0) []
         p1 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p1) []
         i1 = strlen(p1)
@@ -7778,7 +7789,7 @@
         guard_true(i8) []
         i9 = int_le(i22, 255)
         guard_true(i9) []
-        jump(p0, i22, p1)
+        jump(i22, p1)
         """
         expected = """
         [p0, i22, p1]
@@ -7798,6 +7809,7 @@
         """
         short = """
         [p0]
+        guard_nonnull(p0) []
         p1 = getfield_gc_r(p0, descr=nextdescr)
         guard_nonnull(p1) []
         i1 = unicodelen(p1)
@@ -7806,7 +7818,7 @@
         i22 = unicodegetitem(p1, 7, descr=)
         i8 = int_ge(i22, 0)
         guard_true(i8) []
-        jump(p0, i22, p1)
+        jump(i22, p1)
         """
         expected = """
         [p0, i22, p1]
@@ -7906,6 +7918,7 @@
         """
         short = """
         [p0]
+        guard_nonnull(p0) []
         i0 = getfield_gc_i(p0, descr=valuedescr)
         guard_value(i0, 1) []
         jump()
@@ -8004,9 +8017,9 @@
         expected = """
         [p5, p6, i14, i12]
         i13 = int_add(i14, 7)
-        call_n(i12, i13, descr=nonwritedescr)
+        call_n(i13, i12, descr=nonwritedescr)
         setfield_gc(p6, i14, descr=chardescr)
-        jump(p5, p6, i14, i12)
+        jump(p5, p6, i14, i13)
         """
         self.optimize_loop(ops, expected)
 
@@ -8055,10 +8068,10 @@
     def test_constptr_constptr_getfield_setfield(self):
         ops = """
         [p0]
-        p10 = getfield_gc_r(ConstPtr(myptr), descr=valuedescr)
+        p10 = getfield_gc_r(ConstPtr(myptr), descr=otherdescr)
         guard_value(p10, ConstPtr(myptr2)) []
         call_n(p10, descr=nonwritedescr)
-        setfield_gc(ConstPtr(myptr), ConstPtr(myptr2), descr=valuedescr)
+        setfield_gc(ConstPtr(myptr), ConstPtr(myptr2), descr=otherdescr)
         jump(p0)
         """
         expected = """
@@ -8086,10 +8099,10 @@
     def test_box_constptr_getfield_setfield(self):
         ops = """
         [p0]
-        p10 = getfield_gc_r(p0, descr=valuedescr)
+        p10 = getfield_gc_r(p0, descr=otherdescr)
         guard_value(p10, ConstPtr(myptr2)) []
         call_n(p10, descr=nonwritedescr)
-        setfield_gc(p0, ConstPtr(myptr2), descr=valuedescr)
+        setfield_gc(p0, ConstPtr(myptr2), descr=otherdescr)
         jump(p0)
         """
         expected = """
@@ -8102,10 +8115,10 @@
     def test_import_constants_when_folding_pure_operations(self):
         ops = """
         [p0]
-        f1 = getfield_gc_f(p0, descr=valuedescr)
+        f1 = getfield_gc_f(p0, descr=floatdescr)
         f2 = float_abs(f1)
         call_n(7.0, descr=nonwritedescr)
-        setfield_gc(p0, -7.0, descr=valuedescr)
+        setfield_gc(p0, -7.0, descr=floatdescr)
         jump(p0)
         """
         expected = """
@@ -8141,6 +8154,7 @@
         self.optimize_loop(ops, expected, expected_short=short)
 
     def test_prioritize_getfield1(self):
+        py.test.skip("we no longer do it, and while unfortunate I don't think it's that relevant")
         ops = """
         [p1, p2]
         i1 = getfield_gc_i(p1, descr=valuedescr)
@@ -8265,6 +8279,7 @@
         """
         short = """
         [p22, p18]
+        guard_nonnull(p22) []
         i1 = getfield_gc_i(p22, descr=valuedescr)
         guard_value(i1, 2) []
         jump()
@@ -8296,6 +8311,7 @@
         """
         short = """
         [p22, p18, i1]
+        guard_nonnull(p22) []
         i2 = getfield_gc_i(p22, descr=valuedescr)
         jump(i2)
         """
@@ -8341,6 +8357,7 @@
         call_n(p2, descr=nonwritedescr)
         p3 = new_with_vtable(descr=nodesize)
         setarrayitem_gc(p1, 3, p3, descr=gcarraydescr)
+        ifoo = arraylen_gc(p1, descr=gcarraydescr) # killed
         jump(p1, p3)
         """
         self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -118,7 +118,9 @@
     node2.parent.parent.typeptr = node_vtable2
     node2addr = lltype.cast_opaque_ptr(llmemory.GCREF, node2)
     myptr = lltype.cast_opaque_ptr(llmemory.GCREF, node)
-    myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(NODE))
+    mynode2 = lltype.malloc(NODE)
+    mynode2.parent.typeptr = node_vtable
+    myptr2 = lltype.cast_opaque_ptr(llmemory.GCREF, mynode2)
     nullptr = lltype.nullptr(llmemory.GCREF.TO)
     #nodebox2 = InputArgRef(lltype.cast_opaque_ptr(llmemory.GCREF, node2))
     nodesize = cpu.sizeof(NODE, node_vtable)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,6 +6,7 @@
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization, LoopInfo, MININT, MAXINT
+from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
 from rpython.jit.metainterp.resoperation import rop, ResOperation, GuardResOp
@@ -53,6 +54,10 @@
                 arr_info = info.ArrayPtrInfo(preamble_info.arraydescr)
                 arr_info.lenbound = preamble_info.getlenbound(None)
                 op.set_forwarded(arr_info)
+            if isinstance(preamble_info, StrPtrInfo):
+                str_info = StrPtrInfo(preamble_info.mode)
+                str_info.lenbound = preamble_info.getlenbound(None)
+                op.set_forwarded(str_info)
             if preamble_info.is_nonnull():
                 self.make_nonnull(op)
         elif isinstance(preamble_info, intutils.IntBound):
@@ -111,6 +116,7 @@
         label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
         target_token = self.finalize_short_preamble(label_op,
                                                     state.virtual_state)
+        self.main_target_token = target_token
         label_op.setdescr(target_token)
         extra = self.short_preamble_producer.used_boxes
         label_op.initarglist(label_args + extra)
@@ -212,11 +218,15 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            short_preamble = target_token.short_preamble
             pass_to_short = target_virtual_state.make_inputargs(args,
                 self.optimizer, append_virtuals=True)
             args = target_virtual_state.make_inputargs(args,
                 self.optimizer)
+            if target_token is self.main_target_token:
+                # rebuild the short preamble, it might have changed
+                new_sp = self.short_preamble_producer.build_short_preamble()
+                target_token.short_preamble = new_sp
+            short_preamble = target_token.short_preamble
             extra = self.inline_short_preamble(pass_to_short, args,
                 short_preamble[0].getarglist(), short_preamble,
                 short_preamble[-1].getarglist(), self.optimizer.patchguardop)
@@ -257,8 +267,11 @@
                 op.set_forwarded(None)
 
     def export_state(self, start_label, original_label_args, renamed_inputargs):
-        virtual_state = self.get_virtual_state(original_label_args)
-        end_args = [self.get_box_replacement(a) for a in original_label_args]
+        self.optimizer.force_at_end_of_preamble()
+        end_args = [self.optimizer.force_box_for_end_of_preamble(a)
+                    for a in original_label_args]
+        virtual_state = self.get_virtual_state(end_args)
+        end_args = [self.get_box_replacement(arg) for arg in end_args]
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -609,9 +609,6 @@
             pass
         if box.type == 'r':
             info = opt.getptrinfo(box)
-            if info is not None:
-                rec = {}
-                box = info.force_at_the_end_of_preamble(box, opt.optimizer, rec)
             if info is not None and info.is_virtual():
                 result = info.visitor_dispatch_virtual_type(self)
                 self.info[box] = result
@@ -629,7 +626,6 @@
         return result
 
     def get_virtual_state(self, jump_args):
-        self.optimizer.force_at_end_of_preamble()
         if self.optimizer.optearlyforce:
             opt = self.optimizer.optearlyforce
         else:
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -93,7 +93,7 @@
                 optforce.get_box_replacement(op).set_forwarded(c_s)
                 return c_s
         self._is_virtual = False
-        lengthbox = self.getstrlen(op, optforce, self.mode)
+        lengthbox = self.getstrlen(op, optforce.optstring, self.mode)
         newop = ResOperation(self.mode.NEWSTR, [lengthbox])
         if not we_are_translated():
             newop.name = 'FORCE'
@@ -122,16 +122,26 @@
         string_optimizer.emit_operation(lengthop)
         return lengthop
 
+    def make_guards(self, op, short):
+        info.AbstractVirtualPtrInfo.make_guards(self, op, short)
+        if self.lenbound and self.lenbound.lower >= 1:
+            if self.mode is mode_string:
+                lenop = ResOperation(rop.STRLEN, [op])
+            elif self.mode is mode_unicode:
+                lenop = ResOperation(rop.UNICODELEN, [op])
+            short.append(lenop)
+            self.lenbound.make_guards(lenop, short)
+
     def string_copy_parts(self, op, string_optimizer, targetbox, offsetbox,
                           mode):
-         # Copies the pointer-to-string 'self' into the target string
-         # given by 'targetbox', at the specified offset.  Returns the offset
-         # at the end of the copy.
-         lengthbox = self.getstrlen(op, string_optimizer, mode)
-         srcbox = self.force_box(op, string_optimizer)
-         return copy_str_content(string_optimizer, srcbox, targetbox,
-                                 CONST_0, offsetbox, lengthbox, mode)
-
+        # Copies the pointer-to-string 'self' into the target string
+        # given by 'targetbox', at the specified offset.  Returns the offset
+        # at the end of the copy.
+        lengthbox = self.getstrlen(op, string_optimizer, mode)
+        srcbox = self.force_box(op, string_optimizer)
+        return copy_str_content(string_optimizer, srcbox, targetbox,
+                                CONST_0, offsetbox, lengthbox, mode)
+     
 class VStringPlainInfo(StrPtrInfo):
     #_attrs_ = ('mode', '_is_virtual')
 
@@ -398,6 +408,9 @@
 class OptString(optimizer.Optimization):
     "Handling of strings and unicodes."
 
+    def setup(self):
+        self.optimizer.optstring = self
+
     def make_vstring_plain(self, op, mode, length):
         vvalue = VStringPlainInfo(mode, True, length)
         op = self.replace_op_with(op, op.getopnum())

From noreply at buildbot.pypy.org  Wed Aug 26 22:32:13 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Aug 2015 22:32:13 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: ups fix some tests
Message-ID: <20150826203213.A70801C0EFC@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79234:4aca476e6954
Date: 2015-08-26 22:32 +0200
http://bitbucket.org/pypy/pypy/changeset/4aca476e6954/

Log:	ups fix some tests

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -669,6 +669,7 @@
                 IntLowerBound
 
         if mode is None:
+            # XXX we can do better if we know it's an array
             return IntLowerBound(0)
         else:
             return ConstIntBound(self.getstrlen(None, None, mode).getint())
diff --git a/rpython/jit/metainterp/optimizeopt/intbounds.py b/rpython/jit/metainterp/optimizeopt/intbounds.py
--- a/rpython/jit/metainterp/optimizeopt/intbounds.py
+++ b/rpython/jit/metainterp/optimizeopt/intbounds.py
@@ -431,8 +431,9 @@
         v2 = self.getptrinfo(op.getarg(0))
         intbound = self.getintbound(op.getarg(1))
         if (intbound.has_lower and v2 is not None and
-            v2.getlenbound(v2.mode) is not None):
-            v2.getlenbound(v2.mode).make_ge(IntLowerBound(intbound.lower + 1))
+            v2.getlenbound(vstring.mode_string) is not None):
+            lb = IntLowerBound(intbound.lower + 1)
+            v2.getlenbound(vstring.mode_string).make_ge(lb)
         v1.make_ge(IntLowerBound(0))
         v1.make_lt(IntUpperBound(256))
 
@@ -474,8 +475,9 @@
         v2 = self.getptrinfo(op.getarg(0))
         intbound = self.getintbound(op.getarg(1))
         if (intbound.has_lower and v2 is not None and
-            v2.getlenbound(v2.mode) is not None):
-            v2.getlenbound(v2.mode).make_ge(IntLowerBound(intbound.lower + 1))
+            v2.getlenbound(vstring.mode_unicode) is not None):
+            lb = IntLowerBound(intbound.lower + 1)
+            v2.getlenbound(vstring.mode_unicode).make_ge(lb)
 
     def make_int_lt(self, box1, box2):
         b1 = self.getintbound(box1)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -3724,6 +3724,7 @@
         self.optimize_loop(ops, expected)
 
     def test_int_add_sub_constants_inverse(self):
+        py.test.skip("reenable")
         import sys
         ops = """
         [i0, i10, i11, i12, i13]
@@ -4257,6 +4258,7 @@
         strsetitem(p3, i2, i0)
         i5 = int_add(i2, 1)
         strsetitem(p3, i5, i1)
+        ifoo = int_add(i5, 1)
         jump(i1, i0, p3)
         """
         self.optimize_strunicode_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -77,6 +77,7 @@
     distinction anymore)"""
 
     short_preamble_producer = None
+    main_target_token = None
 
     def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,

From noreply at buildbot.pypy.org  Wed Aug 26 22:52:37 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 26 Aug 2015 22:52:37 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: merge default into branch
Message-ID: <20150826205237.238711C0684@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79235:225756b155a3
Date: 2015-08-26 23:52 +0300
http://bitbucket.org/pypy/pypy/changeset/225756b155a3/

Log:	merge default into branch

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -353,8 +353,7 @@
 Except when otherwise stated (look for LICENSE files or copyright/license
 information at the beginning of each file) the files in the 'lib-python/2.7'
 directory are all copyrighted by the Python Software Foundation and licensed
-under the Python Software License of which you can find a copy here:
-http://www.python.org/doc/Copyright.html 
+under the terms that you can find here: https://docs.python.org/2/license.html
 
 License for 'pypy/module/unicodedata/'
 ======================================
@@ -436,4 +435,4 @@
 
 The code is based on gperftools. You may see a copy of the License for it at
 
-    https://code.google.com/p/gperftools/source/browse/COPYING
+    https://github.com/gperftools/gperftools/blob/master/COPYING
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -31,15 +31,14 @@
   and add the new file to  pypy/doc/index-of-whatsnew.rst
 * go to pypy/tool/release and run
   ``force-builds.py ``
-  The following binaries should be built, however, we need more buildbots
- - JIT: windows, linux, os/x, armhf, armel
- - no JIT: windows, linux, os/x
- - sandbox: linux, os/x
+  The following JIT binaries should be built, however, we need more buildbots
+  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+  freebsd64 
 
 * wait for builds to complete, make sure there are no failures
 * download the builds, repackage binaries. Tag the release version
   and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tools to do this. 
+  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
 
   Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
   and to cobra, as some packagers prefer a clearly labeled source package
diff --git a/pypy/module/_multiprocessing/test/test_memory.py b/pypy/module/_multiprocessing/test/test_memory.py
--- a/pypy/module/_multiprocessing/test/test_memory.py
+++ b/pypy/module/_multiprocessing/test/test_memory.py
@@ -1,8 +1,12 @@
+import sys
+
 class AppTestMemory:
     spaceconfig = dict(usemodules=('_multiprocessing', 'mmap',
                                    '_rawffi', 'itertools',
-                                   'signal', 'select', 'fcntl',
+                                   'signal', 'select',
                                    'binascii'))
+    if sys.platform != 'win32':
+        spaceconfig['usemodules'] += ('fcntl',)
 
     def test_address_of(self):
         import _multiprocessing
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -1,12 +1,19 @@
+import sys
+
 from pypy.module._multiprocessing.interp_semaphore import (
     RECURSIVE_MUTEX, SEMAPHORE)
 
 
 class AppTestSemaphore:
     spaceconfig = dict(usemodules=('_multiprocessing', 'thread',
-                                   'signal', 'select', 'fcntl',
+                                   'signal', 'select',
                                    'binascii', 'struct'))
 
+    if sys.platform == 'win32':
+        spaceconfig['usemodules'] += ('_rawffi',)
+    else:
+        spaceconfig['usemodules'] += ('fcntl',)
+
     def setup_class(cls):
         cls.w_SEMAPHORE = cls.space.wrap(SEMAPHORE)
         cls.w_RECURSIVE = cls.space.wrap(RECURSIVE_MUTEX)
diff --git a/pypy/tool/release/force-builds.py b/pypy/tool/release/force-builds.py
--- a/pypy/tool/release/force-builds.py
+++ b/pypy/tool/release/force-builds.py
@@ -28,6 +28,7 @@
 #    'pypy-c-app-level-win-x86-32',
     'pypy-c-jit-linux-x86-32',
     'pypy-c-jit-linux-x86-64',
+    'pypy-c-jit-freebsd-9-x86-64',
     'pypy-c-jit-macosx-x86-64',
     'pypy-c-jit-win-x86-32',
     'build-pypy-c-jit-linux-armhf-raring',
@@ -42,7 +43,7 @@
         import pwd
         return pwd.getpwuid(os.getuid())[0]
 
-def main(branch, server):
+def main(branch, server, user):
     #XXX: handle release tags
     #XXX: handle validity checks
     lock = defer.DeferredLock()
@@ -56,7 +57,7 @@
         print 'Forcing', builder, '...'
         url = "http://" + server + "/builders/" + builder + "/force"
         args = [
-            ('username', get_user()),
+            ('username', user),
             ('revision', ''),
             ('forcescheduler', 'Force Scheduler'),
             ('submit', 'Force Build'),
@@ -78,7 +79,8 @@
     parser = optparse.OptionParser()
     parser.add_option("-b", "--branch", help="branch to build", default='')
     parser.add_option("-s", "--server", help="buildbot server", default="buildbot.pypy.org")
+    parser.add_option("-u", "--user", help="user name to report", default=get_user())
     (options, args) = parser.parse_args()
     if  not options.branch:
         parser.error("branch option required")
-    main(options.branch, options.server)
+    main(options.branch, options.server, user=options.user)
diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,12 +1,12 @@
 # Edit these appropriately before running this script
 maj=2
 min=6
-rev=0
+rev=1
 # This script will download latest builds from the buildmaster, rename the top
 # level directory, and repackage ready to be uploaded to bitbucket. It will also
 # download source, assuming a tag for the release already exists, and repackage them.
 
-for plat in linux linux64 linux-armhf-raspbian linux-armhf-raring linux-armel osx64
+for plat in linux linux64 linux-armhf-raspbian linux-armhf-raring linux-armel osx64 freebsd64
   do
     wget http://buildbot.pypy.org/nightly/release-$maj.$min.x/pypy-c-jit-latest-$plat.tar.bz2
     tar -xf pypy-c-jit-latest-$plat.tar.bz2
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -631,9 +631,11 @@
         raise Exception("TreeLoop.token is killed")
     token = property(_token, _token)
 
-    # This is the jitcell where the trace starts. Labels within the trace might
-    # belong to some other jitcells in the sens that jumping to this other
-    # jitcell will result in a jump to the label.
+    # This is the jitcell where the trace starts.  Labels within the
+    # trace might belong to some other jitcells, i.e. they might have
+    # TargetTokens with a different value for 'targeting_jitcell_token'.
+    # But these TargetTokens also have a 'original_jitcell_token' field,
+    # which must be equal to this one.
     original_jitcell_token = None
 
     def __init__(self, name):

From noreply at buildbot.pypy.org  Wed Aug 26 23:10:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Wed, 26 Aug 2015 23:10:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix?
Message-ID: <20150826211026.BE31E1C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79236:b405a035dfcb
Date: 2015-08-26 23:10 +0200
http://bitbucket.org/pypy/pypy/changeset/b405a035dfcb/

Log:	fix?

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -2739,8 +2739,8 @@
             '''
             p1.nextdescr = p2
             where p2 is a node_vtable, valuedescr=i2
-            ''', rop.GUARD_TRUE, values=[InputArgInt(0),
-                                         InputArgRef(self.nodeaddr)])
+            ''', rop.GUARD_TRUE, values=[InputArgRef(self.nodeaddr),
+                                         InputArgInt(0)])
 
     def test_expand_fail_lazy_setfield_2(self):
         ops = """

From noreply at buildbot.pypy.org  Wed Aug 26 23:33:49 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Wed, 26 Aug 2015 23:33:49 +0200 (CEST)
Subject: [pypy-commit] pypy default: cleanup old freebsd versions,
 future-proof newer ones, fixes issue #2126
Message-ID: <20150826213349.E8BF51C05CE@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79237:6edeb840fb4b
Date: 2015-08-27 00:16 +0300
http://bitbucket.org/pypy/pypy/changeset/6edeb840fb4b/

Log:	cleanup old freebsd versions, future-proof newer ones, fixes issue
	#2126

diff --git a/lib_pypy/ctypes_support.py b/lib_pypy/ctypes_support.py
--- a/lib_pypy/ctypes_support.py
+++ b/lib_pypy/ctypes_support.py
@@ -28,7 +28,7 @@
     def _where_is_errno():
         return standard_c_lib.__errno_location()
 
-elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'):
+elif sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
     standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int)
     standard_c_lib.__error.argtypes = None
     def _where_is_errno():
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -116,9 +116,6 @@
 class AppTestLockSignals(GenericTestThread):
     pytestmark = py.test.mark.skipif("os.name != 'posix'")
 
-    def setup_class(cls):
-        cls.w_using_pthread_cond = cls.space.wrap(sys.platform == 'freebsd6')
-
     def w_acquire_retries_on_intr(self, lock):
         import thread, os, signal, time
         self.sig_recvd = False
@@ -157,8 +154,6 @@
         raise KeyboardInterrupt
 
     def test_lock_acquire_interruption(self):
-        if self.using_pthread_cond:
-            skip('POSIX condition variables cannot be interrupted')
         import thread, signal, time
         # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck
         # in a deadlock.
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -32,7 +32,6 @@
 _WIN32 = _MSVC or _MINGW
 _WIN64 = _WIN32 and is_emulated_long
 _MAC_OS = platform.name == "darwin"
-_FREEBSD_7 = platform.name == "freebsd7"
 
 _LITTLE_ENDIAN = sys.byteorder == 'little'
 _BIG_ENDIAN = sys.byteorder == 'big'

From noreply at buildbot.pypy.org  Thu Aug 27 01:21:15 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 01:21:15 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: start reworking virtualstate
Message-ID: <20150826232115.EF6DC1C088E@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79238:ebc71358494b
Date: 2015-08-27 01:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ebc71358494b/

Log:	start reworking virtualstate

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -202,14 +202,13 @@
         jitcelltoken = jump_op.getdescr()
         virtual_state = self.get_virtual_state(jump_op.getarglist())
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
-        infos = [self.optimizer.getinfo(arg) for arg in args]
         for target_token in jitcelltoken.target_tokens:
             target_virtual_state = target_token.virtual_state
             if target_virtual_state is None:
                 continue
             try:
                 extra_guards = target_virtual_state.generate_guards(
-                    virtual_state, args, infos, self.optimizer.cpu)
+                    virtual_state, args, jump_op.getarglist(), self.optimizer)
                 patchguardop = self.optimizer.patchguardop
                 for guard in extra_guards.extra_guards:
                     if isinstance(guard, GuardResOp):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,13 +1,11 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
-from rpython.jit.metainterp.history import (ConstInt, Const,
-        ConstPtr, ConstFloat)
+from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt import info
 from rpython.jit.metainterp.optimizeopt.intutils import \
-     ConstIntBound, MININT, MAXINT, IntBound
+     MININT, MAXINT, IntBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
-     AbstractInputArg
-from rpython.rlib.debug import debug_start, debug_stop, debug_print
-from rpython.rlib.objectmodel import we_are_translated
+     InputArgInt, InputArgRef, InputArgFloat
+from rpython.rlib.debug import debug_print
 
 LEVEL_UNKNOWN = '\x00'
 LEVEL_NONNULL = '\x01'
@@ -22,16 +20,11 @@
         self.msg = msg
         self.state = state
 
-def get_forwarded(box):
-    if not isinstance(box, Const):
-        return box.get_forwarded()
-    if box.type == 'i':
-        return ConstIntBound(box.getint())
-    xxx
 
 class GenerateGuardState(object):
-    def __init__(self, cpu=None, guards=None, renum=None, bad=None):
-        self.cpu = cpu
+    def __init__(self, optimizer=None, guards=None, renum=None, bad=None):
+        self.optimizer = optimizer
+        self.cpu = optimizer.cpu
         if guards is None:
             guards = []
         self.extra_guards = guards
@@ -42,20 +35,29 @@
             bad = {}
         self.bad = bad
 
+    def get_runtime_field(self, box, descr):
+        if descr.is_pointer_field():
+            xxx
+        elif descr.is_float_field():
+            yyy
+        else:
+            struct = box.getref_base()
+            return InputArgInt(self.cpu.bh_getfield_gc_i(struct, descr))
+
 class AbstractVirtualStateInfo(object):
     position = -1
 
-    def generate_guards(self, other, op, opinfo, state):
+    def generate_guards(self, other, op, runtime_op, state):
         """ generate guards (output in the list extra_guards) that make runtime
         values of the shape other match the shape of self. if that's not
         possible, VirtualStatesCantMatch is thrown and bad gets keys set which
         parts of the state are the problem.
 
-        the function can peek into opinfo (and particularly also the op)
+        the function can peek into the information about the op, as well
+        as runtime value (passed in runtime_op)
         as a guiding heuristic whether making such guards makes
         sense. if None is passed in for op, no guard is ever generated, and
         this function degenerates to a generalization check."""
-        assert opinfo is None or opinfo.is_constant() or isinstance(opinfo, info.AbstractInfo)
         assert self.position != -1
         if self.position in state.renum:
             if state.renum[self.position] != other.position:
@@ -69,14 +71,14 @@
         else:
             state.renum[self.position] = other.position
             try:
-                self._generate_guards(other, op, opinfo, state)
+                self._generate_guards(other, op, runtime_op, state)
             except VirtualStatesCantMatch, e:
                 state.bad[self] = state.bad[other] = None
                 if e.state is None:
                     e.state = state
                 raise e
 
-    def _generate_guards(self, other, value, state):
+    def _generate_guards(self, other, box, runtime_box, state):
         raise VirtualStatesCantMatch(
                 'Generating guards for making the VirtualStates ' +
                 'at hand match have not been implemented')
@@ -114,14 +116,15 @@
     def __init__(self, fielddescrs):
         self.fielddescrs = fielddescrs
 
-    def _generate_guards(self, other, box, opinfo, state):
+    def _generate_guards(self, other, box, runtime_box, state):
         if not self._generalization_of_structpart(other):
             raise VirtualStatesCantMatch("different kinds of structs")
 
         assert isinstance(other, AbstractVirtualStructStateInfo)
         assert len(self.fielddescrs) == len(self.fieldstate)
         assert len(other.fielddescrs) == len(other.fieldstate)
-        if box is not None:
+        opinfo = state.optimizer.getptrinfo(box)
+        if runtime_box is not None:
             assert opinfo.is_virtual()
 
         if len(self.fielddescrs) != len(other.fielddescrs):
@@ -130,21 +133,18 @@
         for i in range(len(self.fielddescrs)):
             if other.fielddescrs[i] is not self.fielddescrs[i]:
                 raise VirtualStatesCantMatch("field descrs don't match")
-            if box is not None:
+            if runtime_box is not None:
                 fieldbox = opinfo._fields[self.fielddescrs[i].get_index()]
                 # must be there
-                if fieldbox is not None:
-                    fieldinfo = get_forwarded(fieldbox)
-                else:
-                    fieldinfo = None
+                fieldbox_runtime = state.get_runtime_field(runtime_box,
+                                                           self.fielddescrs[i])
             else:
                 fieldbox = None
-                fieldinfo = None
-            # XXX all those ifs are for tests, not sure what to do
+                fieldbox_runtime = None
             if self.fieldstate[i] is not None:
                 self.fieldstate[i].generate_guards(other.fieldstate[i],
                                                    fieldbox,
-                                                   fieldinfo, state)
+                                                   fieldbox_runtime, state)
 
 
     def _generalization_of_structpart(self, other):
@@ -205,6 +205,7 @@
         self.arraydescr = arraydescr
 
     def _generate_guards(self, other, box, opinfo, state):
+        xxx
         if not isinstance(other, VArrayStateInfo):
             raise VirtualStatesCantMatch("other is not an array")
         if self.arraydescr is not other.arraydescr:
@@ -251,6 +252,7 @@
         self.fielddescrs = fielddescrs
 
     def _generate_guards(self, other, box, opinfo, state):
+        xxxx
         if not isinstance(other, VArrayStructStateInfo):
             raise VirtualStatesCantMatch("other is not an VArrayStructStateInfo")
         if self.arraydescr is not other.arraydescr:
@@ -344,7 +346,7 @@
     def is_virtual(self):
         return False
 
-    def _generate_guards(self, other, box, opinfo, state):
+    def _generate_guards(self, other, box, runtime_box, state):
         if self.is_opaque:
             box = None # generating guards for opaque pointers isn't safe
         # XXX This will always retrace instead of forcing anything which
@@ -365,7 +367,7 @@
         if self.level == LEVEL_UNKNOWN:
             # confusingly enough, this is done also for pointers
             # which have the full range as the "bound", so it always works
-            return self._generate_guards_intbounds(other, box, opinfo,
+            return self._generate_guards_intbounds(other, box, runtime_box,
                                                    extra_guards)
 
         # the following conditions often peek into the runtime value that the
@@ -375,7 +377,7 @@
         # will always generate correct behaviour, but performance will differ.
         elif self.level == LEVEL_NONNULL:
             if other.level == LEVEL_UNKNOWN:
-                if box is not None and box.nonnull():
+                if runtime_box is not None and runtime_box.nonnull():
                     op = ResOperation(rop.GUARD_NONNULL, [box], None)
                     extra_guards.append(op)
                     return
@@ -394,15 +396,16 @@
 
         elif self.level == LEVEL_KNOWNCLASS:
             if other.level == LEVEL_UNKNOWN:
-                if (box and box.nonnull() and
-                        self.known_class.same_constant(cpu.ts.cls_of_box(box))):
+                if (runtime_box and runtime_box.nonnull() and
+              self.known_class.same_constant(cpu.ts.cls_of_box(runtime_box))):
                     op = ResOperation(rop.GUARD_NONNULL_CLASS, [box, self.known_class], None)
                     extra_guards.append(op)
                     return
                 else:
                     raise VirtualStatesCantMatch("other's class is unknown")
             elif other.level == LEVEL_NONNULL:
-                if box and self.known_class.same_constant(cpu.ts.cls_of_box(box)):
+                if (runtime_box and self.known_class.same_constant(
+                        cpu.ts.cls_of_box(runtime_box))):
                     op = ResOperation(rop.GUARD_CLASS, [box, self.known_class], None)
                     extra_guards.append(op)
                     return
@@ -426,7 +429,7 @@
                 if self.constbox.same_constant(other.constbox):
                     return
                 raise VirtualStatesCantMatch("different constants")
-            if box is not None and self.constbox.same_constant(box.constbox()):
+            if runtime_box is not None and self.constbox.same_constant(runtime_box.constbox()):
                 op = ResOperation(rop.GUARD_VALUE, [box, self.constbox], None)
                 extra_guards.append(op)
                 return
@@ -434,12 +437,13 @@
                 raise VirtualStatesCantMatch("other not constant")
         assert 0, "unreachable"
 
-    def _generate_guards_intbounds(self, other, box, opinfo, extra_guards):
+    def _generate_guards_intbounds(self, other, box, runtime_box, extra_guards):
         if self.intbound is None:
             return
         if self.intbound.contains_bound(other.intbound):
             return
-        if (box is not None and self.intbound.contains(box.getint())):
+        if (runtime_box is not None and
+            self.intbound.contains(runtime_box.getint())):
             # this may generate a few more guards than needed, but they are
             # optimized away when emitting them
             self.intbound.make_guards(box, extra_guards)
@@ -523,12 +527,13 @@
             return False
         return True
 
-    def generate_guards(self, other, boxes, infos, cpu):
-        assert len(self.state) == len(other.state) == len(boxes) == len(infos)
-        state = GenerateGuardState(cpu)
+    def generate_guards(self, other, boxes, runtime_boxes, optimizer):
+        assert (len(self.state) == len(other.state) == len(boxes) ==
+                len(runtime_boxes))
+        state = GenerateGuardState(optimizer)
         for i in range(len(self.state)):
-            self.state[i].generate_guards(other.state[i], boxes[i], infos[i],
-                                          state)
+            self.state[i].generate_guards(other.state[i], boxes[i],
+                                          runtime_boxes[i], state)
         return state
 
     def make_inputargs(self, inputargs, optimizer, force_boxes=False,
@@ -578,24 +583,6 @@
     def already_seen_virtual(self, keybox):
         return keybox in self.fieldboxes
 
-    #def state(self, box):
-    #    xxx
-    #    value = self.getvalue(box)
-    #    box = value.get_key_box()
-    #    try:
-    #        info = self.info[box]
-    #    except KeyError:
-    #        self.info[box] = info = value.visitor_dispatch_virtual_type(self)
-    #        if value.is_virtual():
-    #            flds = self.fieldboxes[box]
-    #            info.fieldstate = [self.state_or_none(b, value) for b in flds]
-    #    return info
-
-    #def state_or_none(self, box, value):
-    #    if box is None:
-    #        box = value.get_missing_null_value().box
-    #    return self.state(box)
-
     def create_state_or_none(self, box, opt):
         if box is None:
             return None
diff --git a/rpython/jit/metainterp/test/test_del.py b/rpython/jit/metainterp/test/test_del.py
--- a/rpython/jit/metainterp/test/test_del.py
+++ b/rpython/jit/metainterp/test/test_del.py
@@ -22,7 +22,7 @@
                 n -= 1
             return 42
         self.meta_interp(f, [20])
-        self.check_resops({'call': 4,      # calls to a helper function
+        self.check_resops({'call_r': 4,      # calls to a helper function
                            'guard_no_exception': 4,    # follows the calls
                            'int_sub': 2,
                            'int_gt': 2,
@@ -80,7 +80,7 @@
             return 1
         res = self.meta_interp(f, [20], enable_opts='')
         assert res == 1
-        self.check_resops(call=1)   # for the case B(), but not for the case A()
+        self.check_resops(call_r=1)   # for the case B(), but not for the case A()
 
     def test_keepalive(self):
         py.test.skip("XXX fails")   # hum, I think the test itself is broken
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -190,11 +190,11 @@
         # not cached anyway.
         res = self.meta_interp(f, [100], listops=True)
         assert res == f(50)
-        self.check_resops({'new_array_clear': 2, 'getfield_gc': 2,
+        self.check_resops({'new_array_clear': 2, 'getfield_gc_r': 2,
                            'guard_true': 4, 'jump': 1,
-                           'new_with_vtable': 2, 'getinteriorfield_gc': 2,
+                           'new_with_vtable': 2, 'getinteriorfield_gc_i': 2,
                            'setfield_gc': 14, 'int_gt': 2, 'int_sub': 2,
-                           'call': 10, 'int_ge': 2,
+                           'call_i': 6, 'call_n': 2, 'call_r': 2, 'int_ge': 2,
                            'guard_no_exception': 8, 'new': 2})
 
     def test_unrolling_of_dict_iter(self):

From noreply at buildbot.pypy.org  Thu Aug 27 10:01:36 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 27 Aug 2015 10:01:36 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: fix some issues with finalizers
Message-ID: <20150827080136.7C1DD1C0557@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1947:187d67c77fd4
Date: 2015-08-27 10:05 +0200
http://bitbucket.org/pypy/stmgc/changeset/187d67c77fd4/

Log:	fix some issues with finalizers

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1365,9 +1365,10 @@
         stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
     }
 
+    commit_finalizers();
+
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
-    commit_finalizers();
 
     /* update 'overflow_number' if needed */
     if (STM_PSEGMENT->overflow_number_has_been_used) {
diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -29,6 +29,9 @@
 static void _commit_finalizers(void)
 {
     /* move finalizer lists to g_finalizers for major collections */
+    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
+        spin_loop();
+    }
 
     if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
@@ -60,6 +63,8 @@
 
     free(STM_PSEGMENT->finalizers);
     STM_PSEGMENT->finalizers = NULL;
+
+    __sync_lock_release(&g_finalizers.lock);
 }
 
 static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
@@ -389,7 +394,7 @@
 static void deal_with_objects_with_finalizers(void)
 {
     /* for non-light finalizers */
-
+    assert(_has_mutex());
     /* there is one 'objects_with_finalizers' list per segment.
        Objects that die at a major collection running in the same
        transaction as they were created will be put in the
@@ -495,23 +500,29 @@
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
     /* called between transactions */
-    static int lock = 0;
-
-    if (__sync_lock_test_and_set(&lock, 1) != 0) {
-        /* can't acquire the lock: someone else is likely already
-           running this function, so don't wait. */
-        return;
-    }
-
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
     _stm_start_transaction(tl);
+    /* XXX: become inevitable, bc. otherwise, we would need to keep
+       around the original g_finalizers.run_finalizers to restore it
+       in case of an abort. */
+    _stm_become_inevitable("finalizer-Tx");
 
-    fprintf(stderr, "run_finalizers: %lu\n", list_count(g_finalizers.run_finalizers));
-    _execute_finalizers(&g_finalizers);
+    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        spin_loop();
+    }
+    struct finalizers_s copy = g_finalizers;
+    assert(copy.running_next == NULL);
+    g_finalizers.run_finalizers = NULL;
+    /* others may add to g_finalizers again: */
+    __sync_lock_release(&g_finalizers.lock);
+
+    fprintf(stderr, "run_finalizers: %lu\n", list_count(copy.run_finalizers));
+    _execute_finalizers(©);
 
     _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
 
-    __sync_lock_release(&lock);
+    LIST_FREE(copy.run_finalizers);
 }
diff --git a/c8/stm/finalizer.h b/c8/stm/finalizer.h
--- a/c8/stm/finalizer.h
+++ b/c8/stm/finalizer.h
@@ -1,5 +1,7 @@
 
+/* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
+    long lock;
     struct list_s *objects_with_finalizers;
     uintptr_t count_non_young;
     struct list_s *run_finalizers;
diff --git a/c8/test/test_finalizer.py b/c8/test/test_finalizer.py
--- a/c8/test/test_finalizer.py
+++ b/c8/test/test_finalizer.py
@@ -273,59 +273,3 @@
         self.expect_finalized([])
         stm_major_collect()
         self.expect_finalized([lp1])
-
-
-class TestMoreRegularFinalizers(BaseTest):
-
-    def test_inevitable_in_finalizer(self):
-        lpo = stm_allocate_old(16)
-
-        self._first_time = True
-        @ffi.callback("void(object_t *)")
-        def finalizer(obj):
-            print "finalizing!", obj
-            stm_set_char(lpo, 'a')
-
-            if self._first_time:
-                self._first_time = False
-                # we will switch to the other TX and
-                # make it inevitable, so that our TX
-                # will abort on commit (or validate)
-                self.switch(0, validate=False)
-                self.become_inevitable()
-                self.switch(1, validate=False)
-
-        lib.stmcb_finalizer = finalizer
-        self._finalizer_keepalive = finalizer
-
-        # start a transaction with a finalizing obj
-        self.switch(1)
-        self.start_transaction()
-        lpf = stm_allocate_with_finalizer(16)
-
-        self.push_root(lpf)
-        stm_minor_collect()
-
-
-        self.switch(0)
-        self.start_transaction()
-        stm_set_char(lpo, 'x')
-        self.switch(1)
-        lpf = self.pop_root()
-        # commit this TX, start a new one, let lpf
-        # die with a major-gc:
-        self.commit_transaction()
-        self.start_transaction()
-        stm_major_collect()
-        # commit and run finalizer in separate TX
-        # that will abort because of a conflict
-        self.commit_transaction()
-
-        self.switch(0, validate=False)
-        assert stm_get_char(lpo) == 'x'
-        # commit the now-inevitable TX and run
-        # the aborted finalizer again
-        self.commit_transaction()
-        self.start_transaction()
-        # should now see the value set by finalizer
-        assert stm_get_char(lpo) == 'a'

From noreply at buildbot.pypy.org  Thu Aug 27 10:07:37 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 27 Aug 2015 10:07:37 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: fix
Message-ID: <20150827080737.B4E6D1C05CE@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1948:08c773f2b6ef
Date: 2015-08-27 10:11 +0200
http://bitbucket.org/pypy/stmgc/changeset/08c773f2b6ef/

Log:	fix

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -518,8 +518,9 @@
     /* others may add to g_finalizers again: */
     __sync_lock_release(&g_finalizers.lock);
 
-    fprintf(stderr, "run_finalizers: %lu\n", list_count(copy.run_finalizers));
-    _execute_finalizers(©);
+    if (copy.run_finalizers != NULL) {
+        _execute_finalizers(©);
+    }
 
     _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);

From noreply at buildbot.pypy.org  Thu Aug 27 10:12:36 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 10:12:36 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: write a test that shows a
 problem with heapcache
Message-ID: <20150827081236.4A9BB1C0206@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79239:589909794071
Date: 2015-08-27 10:11 +0200
http://bitbucket.org/pypy/pypy/changeset/589909794071/

Log:	write a test that shows a problem with heapcache

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8735,5 +8735,15 @@
         """
         self.optimize_loop(ops, expected, expected_short=expected_short)
 
+    def test_cached_arrayitem_write_descr(self):
+        ops = """
+        [p0, p1, i0]
+        setarrayitem_gc(p0, 0, i0, descr=arraydescr)
+        ifoo = call_i(descr=writearraydescr)
+        i1 = getarrayitem_gc_i(p1, 0, descr=arraydescr)
+        jump(p0, p1, i1)
+        """
+        self.optimize_loop(ops, ops)
+
 class TestLLtype(OptimizeOptTest, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Thu Aug 27 10:14:17 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 27 Aug 2015 10:14:17 +0200 (CEST)
Subject: [pypy-commit] stmgc use-gcc: remove XXX
Message-ID: <20150827081417.8D1231C0206@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: use-gcc
Changeset: r1949:c8524a717c93
Date: 2015-08-27 10:18 +0200
http://bitbucket.org/pypy/stmgc/changeset/c8524a717c93/

Log:	remove XXX

diff --git a/c8/stm/finalizer.c b/c8/stm/finalizer.c
--- a/c8/stm/finalizer.c
+++ b/c8/stm/finalizer.c
@@ -486,12 +486,7 @@
     LIST_FREE(f->run_finalizers);
 }
 
-/* XXX: can there be a race between _invoke_general_finalizers
-        and _commit_finalizer on g_finalizers (+other places?)?
-   XXX: what happens in _execute_finalizer if the transaction
-        conflicts (or fails to become inevitable) in a finalizer?
-        (the run_finalizers list is half-way cleared?)
-   XXX: according to translator.backendopt.finalizer, getfield_gc
+/* XXX: according to translator.backendopt.finalizer, getfield_gc
         for primitive types is a safe op in light finalizers.
         I don't think that's correct in general (maybe if
         getfield on *dying obj*).

From noreply at buildbot.pypy.org  Thu Aug 27 10:22:44 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 10:22:44 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix
Message-ID: <20150827082244.5A4461C0206@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79240:3c4a45981c62
Date: 2015-08-27 10:15 +0200
http://bitbucket.org/pypy/pypy/changeset/3c4a45981c62/

Log:	fix

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -69,8 +69,9 @@
         else:
             index = g.getarg(1).getint()
             assert index >= 0
+            cf = optheap.arrayitem_cache(descr, index)
             opinfo.setitem(self.getfield_op.getdescr(), index, self.res,
-                           pop, optheap=optheap)
+                           pop, cf, optheap=optheap)
 
     def add_op_to_short(self, sb):
         sop = self.getfield_op

From noreply at buildbot.pypy.org  Thu Aug 27 10:22:46 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 10:22:46 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix clearing of zerod arrays
Message-ID: <20150827082246.7334A1C0206@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79241:7592f7c40713
Date: 2015-08-27 10:22 +0200
http://bitbucket.org/pypy/pypy/changeset/7592f7c40713/

Log:	fix clearing of zerod arrays

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -443,18 +443,20 @@
 
     def _force_elements(self, op, optforce, descr):
         arraydescr = op.getdescr()
+        const = optforce.new_const_item(self.arraydescr)
         for i in range(self.length):
             item = self._items[i]
-            if item is not None:
-                subbox = optforce.force_box(item)
-                setop = ResOperation(rop.SETARRAYITEM_GC,
-                                     [op, ConstInt(i), subbox],
-                                      descr=arraydescr)
-                self._items[i] = None
-                if optforce.optheap is not None:
-                    optforce.optheap.propagate_forward(setop)
-                else:
-                    optforce.emit_operation(setop)
+            if item is None or self._clear and const.same_constant(item):
+                continue
+            subbox = optforce.force_box(item)
+            setop = ResOperation(rop.SETARRAYITEM_GC,
+                                 [op, ConstInt(i), subbox],
+                                  descr=arraydescr)
+            self._items[i] = None
+            if optforce.optheap is not None:
+                optforce.optheap.propagate_forward(setop)
+            else:
+                optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
     def setitem(self, descr, index, struct_op, op, cf=None, optheap=None):

From noreply at buildbot.pypy.org  Thu Aug 27 10:23:31 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 27 Aug 2015 10:23:31 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: import stmgc with finalizer fixes
Message-ID: <20150827082331.054E11C0206@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8-gcc
Changeset: r79242:163412702b2b
Date: 2015-08-27 10:25 +0200
http://bitbucket.org/pypy/pypy/changeset/163412702b2b/

Log:	import stmgc with finalizer fixes

diff --git a/rpython/translator/stm/src_stm/revision b/rpython/translator/stm/src_stm/revision
--- a/rpython/translator/stm/src_stm/revision
+++ b/rpython/translator/stm/src_stm/revision
@@ -1,1 +1,1 @@
-a3cb98b78053
+08c773f2b6ef
diff --git a/rpython/translator/stm/src_stm/stm/core.c b/rpython/translator/stm/src_stm/stm/core.c
--- a/rpython/translator/stm/src_stm/stm/core.c
+++ b/rpython/translator/stm/src_stm/stm/core.c
@@ -1365,9 +1365,10 @@
         stm_rewind_jmp_forget(STM_SEGMENT->running_thread);
     }
 
+    commit_finalizers();
+
     /* XXX do we still need a s_mutex_lock() section here? */
     s_mutex_lock();
-    commit_finalizers();
 
     /* update 'overflow_number' if needed */
     if (STM_PSEGMENT->overflow_number_has_been_used) {
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.c b/rpython/translator/stm/src_stm/stm/finalizer.c
--- a/rpython/translator/stm/src_stm/stm/finalizer.c
+++ b/rpython/translator/stm/src_stm/stm/finalizer.c
@@ -29,6 +29,9 @@
 static void _commit_finalizers(void)
 {
     /* move finalizer lists to g_finalizers for major collections */
+    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
+        spin_loop();
+    }
 
     if (STM_PSEGMENT->finalizers->run_finalizers != NULL) {
         /* copy 'STM_PSEGMENT->finalizers->run_finalizers' into
@@ -60,6 +63,8 @@
 
     free(STM_PSEGMENT->finalizers);
     STM_PSEGMENT->finalizers = NULL;
+
+    __sync_lock_release(&g_finalizers.lock);
 }
 
 static void abort_finalizers(struct stm_priv_segment_info_s *pseg)
@@ -389,7 +394,7 @@
 static void deal_with_objects_with_finalizers(void)
 {
     /* for non-light finalizers */
-
+    assert(_has_mutex());
     /* there is one 'objects_with_finalizers' list per segment.
        Objects that die at a major collection running in the same
        transaction as they were created will be put in the
@@ -481,25 +486,44 @@
     LIST_FREE(f->run_finalizers);
 }
 
+/* XXX: can there be a race between _invoke_general_finalizers
+        and _commit_finalizer on g_finalizers (+other places?)?
+   XXX: what happens in _execute_finalizer if the transaction
+        conflicts (or fails to become inevitable) in a finalizer?
+        (the run_finalizers list is half-way cleared?)
+   XXX: according to translator.backendopt.finalizer, getfield_gc
+        for primitive types is a safe op in light finalizers.
+        I don't think that's correct in general (maybe if
+        getfield on *dying obj*).
+*/
+
 static void _invoke_general_finalizers(stm_thread_local_t *tl)
 {
     /* called between transactions */
-    static int lock = 0;
-
-    if (__sync_lock_test_and_set(&lock, 1) != 0) {
-        /* can't acquire the lock: someone else is likely already
-           running this function, so don't wait. */
-        return;
-    }
-
     rewind_jmp_buf rjbuf;
     stm_rewind_jmp_enterframe(tl, &rjbuf);
     _stm_start_transaction(tl);
+    /* XXX: become inevitable, bc. otherwise, we would need to keep
+       around the original g_finalizers.run_finalizers to restore it
+       in case of an abort. */
+    _stm_become_inevitable("finalizer-Tx");
 
-    _execute_finalizers(&g_finalizers);
+    while (__sync_lock_test_and_set(&g_finalizers.lock, 1) != 0) {
+        /* somebody is adding more finalizers (_commit_finalizer()) */
+        spin_loop();
+    }
+    struct finalizers_s copy = g_finalizers;
+    assert(copy.running_next == NULL);
+    g_finalizers.run_finalizers = NULL;
+    /* others may add to g_finalizers again: */
+    __sync_lock_release(&g_finalizers.lock);
+
+    if (copy.run_finalizers != NULL) {
+        _execute_finalizers(©);
+    }
 
     _stm_commit_transaction();
     stm_rewind_jmp_leaveframe(tl, &rjbuf);
 
-    __sync_lock_release(&lock);
+    LIST_FREE(copy.run_finalizers);
 }
diff --git a/rpython/translator/stm/src_stm/stm/finalizer.h b/rpython/translator/stm/src_stm/stm/finalizer.h
--- a/rpython/translator/stm/src_stm/stm/finalizer.h
+++ b/rpython/translator/stm/src_stm/stm/finalizer.h
@@ -1,5 +1,7 @@
 /* Imported by rpython/translator/stm/import_stmgc.py */
+/* see deal_with_objects_with_finalizers() for explanation of these fields */
 struct finalizers_s {
+    long lock;
     struct list_s *objects_with_finalizers;
     uintptr_t count_non_young;
     struct list_s *run_finalizers;

From noreply at buildbot.pypy.org  Thu Aug 27 10:23:33 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 27 Aug 2015 10:23:33 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: add a TODO about light finalizers
Message-ID: <20150827082333.25CB81C0206@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8-gcc
Changeset: r79243:3744b529b0f0
Date: 2015-08-27 10:26 +0200
http://bitbucket.org/pypy/pypy/changeset/3744b529b0f0/

Log:	add a TODO about light finalizers

diff --git a/TODO b/TODO
--- a/TODO
+++ b/TODO
@@ -1,3 +1,6 @@
+see rpython.translator.backendopt.finalizer.py for an XXX
+about light finalizers
+
 ------------------------------------------------------------
 
 the loop in
diff --git a/rpython/translator/backendopt/finalizer.py b/rpython/translator/backendopt/finalizer.py
--- a/rpython/translator/backendopt/finalizer.py
+++ b/rpython/translator/backendopt/finalizer.py
@@ -47,6 +47,12 @@
         ##     if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':
         ##         # primitive type
         ##         return self.bottom_result()
+        # XXX: for STM, even primitive type getfield requires stm_read()
+        #      and should therefore not be in light finalizers...
+        #      getfield from 'raw' is fine (otherwise there would be a
+        #      become_inevitable() op somewhere); also maybe getfield_gc
+        #      on dying obj is fine (otherwise light finalizers are pretty
+        #      useless).
         if op.opname == 'getfield':
             TP = op.result.concretetype
             if not isinstance(TP, lltype.Ptr) or TP.TO._gckind == 'raw':

From noreply at buildbot.pypy.org  Thu Aug 27 10:23:35 2015
From: noreply at buildbot.pypy.org (Raemi)
Date: Thu, 27 Aug 2015 10:23:35 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c8-gcc: improve timeline in print_stm_log
Message-ID: <20150827082335.2B76E1C0206@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c8-gcc
Changeset: r79244:e15328b24f1b
Date: 2015-08-27 10:26 +0200
http://bitbucket.org/pypy/pypy/changeset/e15328b24f1b/

Log:	improve timeline in print_stm_log

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -331,14 +331,16 @@
     #
     values = stmlog.get_conflicts()
     for c in values[:15]:
-        intervals = 48
+        intervals = 60
         timeline = [0] * intervals
         for t in c.timestamps:
             idx = int((t - start_time) / total_time * intervals)
             timeline[idx] += 1
 
         print str(c)
-        print "time line:", "".join(['x' if i else '.' for i in timeline])
+        max_events = float(max(timeline))+0.1
+        print "time line:", "|"+"".join(['_xX'[int(i / max_events * 3)]
+                                     if i else ' ' for i in timeline])+"|"
         print
 
 

From noreply at buildbot.pypy.org  Thu Aug 27 10:35:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 10:35:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: some fixes to test_dict
Message-ID: <20150827083517.1BA951C0206@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79245:cc1ab08f4f5f
Date: 2015-08-27 10:34 +0200
http://bitbucket.org/pypy/pypy/changeset/cc1ab08f4f5f/

Log:	some fixes to test_dict

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -641,6 +641,6 @@
         # to be non-None, so clearing is not important any more
         return VArrayStateInfo(arraydescr)
 
-    def visit_varraystruct(self, arraydescr, fielddescrs):
+    def visit_varraystruct(self, arraydescr, length, fielddescrs):
         return VArrayStructStateInfo(arraydescr, fielddescrs)
 
diff --git a/rpython/jit/metainterp/test/test_dict.py b/rpython/jit/metainterp/test/test_dict.py
--- a/rpython/jit/metainterp/test/test_dict.py
+++ b/rpython/jit/metainterp/test/test_dict.py
@@ -230,7 +230,7 @@
 
         self.meta_interp(f, [10])
         # XXX should be one getinteriorfield_gc.  At least it's one call.
-        self.check_simple_loop(call=1, getinteriorfield_gc=2,
+        self.check_simple_loop(call_i=1, getinteriorfield_gc_i=2,
                                guard_no_exception=1)
 
     def test_ordered_dict_two_lookups(self):
@@ -251,7 +251,7 @@
 
         self.meta_interp(f, [10])
         # XXX should be one getinteriorfield_gc.  At least it's one call.
-        self.check_simple_loop(call=1, getinteriorfield_gc=2,
+        self.check_simple_loop(call_i=1, getinteriorfield_gc_i=2,
                                guard_no_exception=1)
 
     def test_dict_insert_invalidates_caches(self):
@@ -272,7 +272,7 @@
 
         res = self.meta_interp(f, [10])
         assert res == f(10)
-        self.check_simple_loop(call=5)
+        self.check_simple_loop(call_i=4, call_n=1)
 
     def test_dict_array_write_invalidates_caches(self):
         driver = JitDriver(greens = [], reds = 'auto')
@@ -294,7 +294,7 @@
         exp = f(10)
         res = self.meta_interp(f, [10])
         assert res == exp
-        self.check_simple_loop(call=7)
+        self.check_simple_loop(call_i=5, call_n=2)
 
     def test_dict_double_lookup_2(self):
         driver = JitDriver(greens = [], reds = 'auto')
@@ -313,7 +313,7 @@
 
         res = self.meta_interp(f, [10])
         assert res == f(10)
-        self.check_simple_loop(call=3)
+        self.check_simple_loop(call_i=2, call_n=1)
 
     def test_dict_eq_can_release_gil(self):
         from rpython.rtyper.lltypesystem import lltype, rffi
@@ -345,8 +345,9 @@
 
         res = self.meta_interp(f, [10], listops=True)
         assert res == 2 + 1 + 1 + 1
-        self.check_simple_loop(call_may_force=4,    # ll_dict_lookup_trampoline
-                               call=1) # ll_dict_setitem_lookup_done_trampoline
+        self.check_simple_loop(call_may_force_i=4,
+                              # ll_dict_lookup_trampoline
+                              call_n=1) # ll_dict_setitem_lookup_done_trampoline
 
     def test_bug42(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')
@@ -361,7 +362,7 @@
                     if n in mdict:
                         raise Exception
         self.meta_interp(f, [10])
-        self.check_simple_loop(call_may_force=0, call=4)
+        self.check_simple_loop(call_may_force_i=0, call_i=3, call_n=1)
 
     def test_dict_virtual(self):
         myjitdriver = JitDriver(greens = [], reds = 'auto')
@@ -376,7 +377,7 @@
                 n -= 1
             return len(d)
         self.meta_interp(f, [100])
-        self.check_simple_loop(call_may_force=0, call=0, new=0)
+        self.check_simple_loop(call_may_force_i=0, call_i=0, new=0)
 
 
 class TestLLtype(DictTests, LLJitMixin):

From noreply at buildbot.pypy.org  Thu Aug 27 11:42:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 11:42:24 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix test_optimizeopt for the
	new virtualstate
Message-ID: <20150827094224.30FEF1C050C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79246:975bc5a56e3a
Date: 2015-08-27 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/975bc5a56e3a/

Log:	fix test_optimizeopt for the new virtualstate

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -45,8 +45,18 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize_loop(self, ops, expected, expected_preamble=None,
-                      call_pure_results=None, expected_short=None):
+                      call_pure_results=None, expected_short=None,
+                      jump_values=None):
         loop = self.parse(ops, postprocess=self.postprocess)
+        jump_op = loop.operations[-1]
+        if jump_values is not None:
+            for i, v in enumerate(jump_values):
+                if v is not None:
+                    jump_op.getarg(i).setref_base(v)
+        else:
+            for i, box in enumerate(jump_op.getarglist()):
+                if box.type == 'r' and not box.is_constant():
+                    box.setref_base(self.nodefulladdr)
         if expected != "crash!":
             expected = self.parse(expected)
         if expected_preamble:
@@ -796,7 +806,8 @@
         jump(i1, p3)
         """
         # We cannot track virtuals that survive for more than two iterations.
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                           jump_values=[None, self.nodefulladdr, None])
 
     def test_p123_nested(self):
         ops = """
@@ -829,7 +840,8 @@
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
         """
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                      jump_values=[None, self.nodefulladdr, self.nodefulladdr])
 
     def test_p123_anti_nested(self):
         ops = """
@@ -865,7 +877,8 @@
         setfield_gc(p1, p3sub, descr=nextdescr)
         jump(i1, p1, p3sub, i1)
         """
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                jump_values=[None, self.nodefulladdr, self.nodefulladdr, None])
 
     def test_dont_delay_setfields(self):
         ops = """
@@ -902,7 +915,8 @@
         setfield_gc(p3, i2, descr=valuedescr)
         jump(p3, i2)
         """
-        self.optimize_loop(ops, expected, preamble, expected_short=short)
+        self.optimize_loop(ops, expected, preamble, expected_short=short,
+                           jump_values=[None, self.nodefulladdr])
 
     # ----------
 
@@ -1004,7 +1018,8 @@
         i1 = int_add(i2, i)
         jump(i, i1)
         """
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                           jump_values=[None, self.nodefulladdr])
 
     def test_virtual_float(self):
         ops = """
@@ -1477,7 +1492,7 @@
         p33 = new_array(0, descr=arraydescr)
         setfield_gc(p31, p33, descr=bdescr)
         p35 = new_with_vtable(descr=nodesize)
-        setfield_gc(p35, p31, descr=valuedescr)
+        setfield_gc(p35, p31, descr=nexttupledescr)
         jump(p0, p35)
         """
         expected = """
@@ -1493,7 +1508,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(descr=nodesize)
-        setfield_gc(p35, p31, descr=valuedescr)
+        setfield_gc(p35, p31, descr=nexttupledescr)
         jump(p0, p35)
         """
         expected = """
@@ -1508,7 +1523,7 @@
         guard_isnull(p18) [p0, p8]
         p31 = new(descr=ssize)
         p35 = new_with_vtable(descr=nodesize)
-        setfield_gc(p35, p31, descr=valuedescr)
+        setfield_gc(p35, p31, descr=nexttupledescr)
         jump(p0, p35, p19, p18)
         """
         expected = """
@@ -1553,7 +1568,8 @@
         i2 = int_add(25, i1)
         jump(i2)
         """
-        self.optimize_loop(ops, expected)
+        jump_values = [self.arrayref]
+        self.optimize_loop(ops, expected, jump_values=jump_values)
 
     def test_varray_alloc_and_set(self):
         ops = """
@@ -1651,7 +1667,8 @@
         guard_value(i3, 15) []
         jump(5)
         """
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                           jump_values=[None, self.arrayref])
 
     def test_p123_array(self):
         ops = """
@@ -1677,7 +1694,8 @@
         jump(i1, p1)
         """
         # We cannot track virtuals that survive for more than two iterations.
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                           jump_values=[None, self.arrayref, None])
 
     def test_varray_forced_1(self):
         ops = """
@@ -1720,7 +1738,8 @@
         escape_n(i1)
         jump(i1)
         """
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                           jump_values=[None, self.tupleaddr])
 
     def test_p123_vstruct(self):
         ops = """
@@ -1746,7 +1765,8 @@
         jump(i1, p1)
         """
         # We cannot track virtuals that survive for more than two iterations.
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble,
+                           jump_values=[None, self.tupleaddr, None])
 
     def test_virtual_raw_malloc_basic(self):
         ops = """
@@ -2764,7 +2784,8 @@
         p3 = escape_r()
         jump(i0, p3)
         """
-        self.optimize_loop(ops, expected)
+        self.optimize_loop(ops, expected,
+                           jump_values=[None, self.array2ref])
 
     def test_bug_3(self):
         ops = """
@@ -3284,9 +3305,9 @@
     def test_ovf_guard_in_short_preamble1(self):
         ops = """
         [p8, p11, i24]
-        p26 = new_with_vtable(descr=nodesize)
+        p26 = new(descr=ssize)
         setfield_gc(p26, i24, descr=adescr)
-        i34 = getfield_gc_pure_i(p11, descr=valuedescr)
+        i34 = getfield_gc_pure_i(p11, descr=abisdescr)
         i35 = getfield_gc_pure_i(p26, descr=adescr)
         i36 = int_add_ovf(i34, i35)
         guard_no_overflow() []
@@ -3336,7 +3357,8 @@
         guard_no_overflow() []
         jump(p0, p1, i16, p3, i17, i7, i11)
         """
-        self.optimize_loop(ops, expected)
+        self.optimize_loop(ops, expected,
+                           jump_values=[None, None, self.tupleaddr])
 
     def test_int_and_or_with_zero(self):
         ops = """
@@ -6143,7 +6165,8 @@
         []
         jump()
         """
-        self.optimize_loop(ops, expected, preamble)
+        self.optimize_loop(ops, expected, preamble, jump_values=
+                           [self.immutaddr])
 
     def test_immutable_constantfold_recursive(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -94,11 +94,15 @@
     cpu = runner.LLGraphCPU(None)
 
     NODE = lltype.GcForwardReference()
+    S = lltype.GcForwardReference()
     NODE.become(lltype.GcStruct('NODE', ('parent', OBJECT),
                                         ('value', lltype.Signed),
                                         ('floatval', lltype.Float),
                                         ('charval', lltype.Char),
+                                        ('nexttuple', lltype.Ptr(S)),
                                         ('next', lltype.Ptr(NODE))))
+    S.become(lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed),
+                        ('b', lltype.Ptr(NODE))))
     NODE2 = lltype.GcStruct('NODE2', ('parent', NODE),
                                      ('other', lltype.Ptr(NODE)))
 
@@ -130,6 +134,7 @@
     floatdescr = cpu.fielddescrof(NODE, 'floatval')
     chardescr = cpu.fielddescrof(NODE, 'charval')
     nextdescr = cpu.fielddescrof(NODE, 'next')
+    nexttupledescr = cpu.fielddescrof(NODE, 'nexttuple')
     otherdescr = cpu.fielddescrof(NODE2, 'other')
     valuedescr3 = cpu.fielddescrof(NODE3, 'value')
     nextdescr3 = cpu.fielddescrof(NODE3, 'next')
@@ -164,6 +169,8 @@
     intobj_immut_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     noimmut_intval = cpu.fielddescrof(INTOBJ_NOIMMUT, 'intval')
     immut_intval = cpu.fielddescrof(INTOBJ_IMMUT, 'intval')
+    immut = lltype.malloc(INTOBJ_IMMUT, zero=True)
+    immutaddr = lltype.cast_opaque_ptr(llmemory.GCREF, immut)
     noimmut_descr = cpu.sizeof(INTOBJ_NOIMMUT, intobj_noimmut_vtable)
     immut_descr = cpu.sizeof(INTOBJ_IMMUT, intobj_immut_vtable)
 
@@ -175,12 +182,18 @@
     immut_ptrval = cpu.fielddescrof(PTROBJ_IMMUT, 'ptrval')
 
     arraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Signed))
+    array = lltype.malloc(lltype.GcArray(lltype.Signed), 15, zero=True)
+    arrayref = lltype.cast_opaque_ptr(llmemory.GCREF, array)
+    array2 = lltype.malloc(lltype.GcArray(lltype.Ptr(S)), 15, zero=True)
+    array2ref = lltype.cast_opaque_ptr(llmemory.GCREF, array2)
     gcarraydescr = cpu.arraydescrof(lltype.GcArray(llmemory.GCREF))
     floatarraydescr = cpu.arraydescrof(lltype.GcArray(lltype.Float))
 
     # a GcStruct not inheriting from OBJECT
-    S = lltype.GcStruct('TUPLE', ('a', lltype.Signed), ('abis', lltype.Signed),
-                        ('b', lltype.Ptr(NODE)))
+    tpl = lltype.malloc(S, zero=True)
+    tupleaddr = lltype.cast_opaque_ptr(llmemory.GCREF, tpl)
+    nodefull2 = lltype.malloc(NODE, zero=True)
+    nodefull2addr = lltype.cast_opaque_ptr(llmemory.GCREF, nodefull2)
     ssize = cpu.sizeof(S, None)
     adescr = cpu.fielddescrof(S, 'a')
     abisdescr = cpu.fielddescrof(S, 'abis')
@@ -270,6 +283,11 @@
     u2array = lltype.GcArray(rffi.USHORT)
     u2arraydescr = cpu.arraydescrof(u2array)
 
+    nodefull = lltype.malloc(NODE2, zero=True)
+    nodefull.parent.next = lltype.cast_pointer(lltype.Ptr(NODE), nodefull)
+    nodefull.parent.nexttuple = tpl
+    nodefulladdr = lltype.cast_opaque_ptr(llmemory.GCREF, nodefull)
+
     # array of structs (complex data)
     complexarray = lltype.GcArray(
         lltype.Struct("complex",
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -35,13 +35,22 @@
             bad = {}
         self.bad = bad
 
+    def get_runtime_item(self, box, descr, i):
+        array = box.getref_base()
+        if descr.is_array_of_pointers():
+            return InputArgRef(self.cpu.bh_getarrayitem_gc_r(array, i, descr))
+        elif descr.is_array_of_floats():
+            return InputArgFloat(self.cpu.bh_getarrayitem_gc_f(array, i, descr))
+        else:
+            return InputArgInt(self.cpu.bh_getarrayitem_gc_i(array, i, descr))
+
     def get_runtime_field(self, box, descr):
+        struct = box.getref_base()
         if descr.is_pointer_field():
-            xxx
+            return InputArgRef(self.cpu.bh_getfield_gc_r(struct, descr))
         elif descr.is_float_field():
-            yyy
+            return InputArgFloat(self.cpu.bh_getfield_gc_f(struct, descr))
         else:
-            struct = box.getref_base()
             return InputArgInt(self.cpu.bh_getfield_gc_i(struct, descr))
 
 class AbstractVirtualStateInfo(object):
@@ -204,8 +213,7 @@
     def __init__(self, arraydescr):
         self.arraydescr = arraydescr
 
-    def _generate_guards(self, other, box, opinfo, state):
-        xxx
+    def _generate_guards(self, other, box, runtime_box, state):
         if not isinstance(other, VArrayStateInfo):
             raise VirtualStatesCantMatch("other is not an array")
         if self.arraydescr is not other.arraydescr:
@@ -213,14 +221,16 @@
         if len(self.fieldstate) != len(other.fieldstate):
             raise VirtualStatesCantMatch("other has a different length")
         fieldbox = None
-        fieldinfo = None
+        fieldbox_runtime = None
         for i in range(len(self.fieldstate)):
-            if box is not None:
+            if runtime_box is not None:
+                opinfo = state.optimizer.getptrinfo(box)
                 assert isinstance(opinfo, info.ArrayPtrInfo)
                 fieldbox = opinfo._items[i]
-                fieldinfo = get_forwarded(fieldbox)
+                fieldbox_runtime = state.get_runtime_item(runtime_box,
+                                            self.arraydescr, i)
             self.fieldstate[i].generate_guards(other.fieldstate[i],
-                                            fieldbox, fieldinfo, state)
+                                            fieldbox, fieldbox_runtime, state)
 
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)

From noreply at buildbot.pypy.org  Thu Aug 27 11:43:10 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Aug 2015 11:43:10 +0200 (CEST)
Subject: [pypy-commit] pypy default: Comment to explain what the test is for
Message-ID: <20150827094310.CA0431C050C@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79247:2cfeb200c73c
Date: 2015-08-27 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/2cfeb200c73c/

Log:	Comment to explain what the test is for

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8445,6 +8445,7 @@
         self.optimize_loop(ops, expected)
 
     def test_issue1080_infinitie_loop_virtual(self):
+        # Same comment as the following test_issue1080_infinitie_loop_simple
         ops = """
         [p10]
         p52 = getfield_gc(p10, descr=nextdescr) # inst_storage
@@ -8467,6 +8468,10 @@
         self.raises(InvalidLoop, self.optimize_loop, ops, ops)
 
     def test_issue1080_infinitie_loop_simple(self):
+        # 'quasiimmutdescr' is a QuasiImmutDescr initialized with the
+        # 'quasibox' as the quasi-immutable instance.  We close the loop
+        # with ConstPtr(myptr), i.e. a different pointer.  The test checks
+        # that the resulting loop is invalid.
         ops = """
         [p69]
         quasiimmut_field(p69, descr=quasiimmutdescr)

From noreply at buildbot.pypy.org  Thu Aug 27 11:48:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 11:48:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix test_optimizebridge
Message-ID: <20150827094839.5BDD31C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79248:6f2e5221c6f2
Date: 2015-08-27 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/6f2e5221c6f2/

Log:	fix test_optimizebridge

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebridge.py
@@ -9,8 +9,10 @@
     enable_opts = "intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll"
 
     def optimize(self, ops, bridge_ops, expected, expected_loop=None,
-                 inline_short_preamble=True):
+                 inline_short_preamble=True, jump_values=None,
+                 bridge_values=None):
         loop = self.parse(ops, postprocess=self.postprocess)
+        self.set_values(loop.operations, jump_values)
         if expected_loop is not None:
             xxx
             exp_loop = self.parse(expected_loop, postprocess=self.postprocess)
@@ -28,6 +30,7 @@
         guards = [op for op in loop.operations if op.is_guard()]
         assert len(guards) == 1, "more than one guard in the loop"
         bridge = self.parse(bridge_ops, postprocess=self.postprocess)
+        self.set_values(bridge.operations, bridge_values)
         start_label = ResOperation(rop.LABEL, bridge.inputargs)
         bridge.operations[-1].setdescr(jitcell_token)
         self.add_guard_future_condition(bridge)
@@ -115,7 +118,9 @@
         [p0]
         jump(1, 3)
         """
-        self.optimize(loop, bridge, expected)
+        self.optimize(loop, bridge, expected,
+                      jump_values=[None, self.simpleaddr],
+                      bridge_values=[None, self.simpleaddr])
 
     def test_virtual_state_guard_needed(self):
         pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -48,15 +48,7 @@
                       call_pure_results=None, expected_short=None,
                       jump_values=None):
         loop = self.parse(ops, postprocess=self.postprocess)
-        jump_op = loop.operations[-1]
-        if jump_values is not None:
-            for i, v in enumerate(jump_values):
-                if v is not None:
-                    jump_op.getarg(i).setref_base(v)
-        else:
-            for i, box in enumerate(jump_op.getarglist()):
-                if box.type == 'r' and not box.is_constant():
-                    box.setref_base(self.nodefulladdr)
+        self.set_values(loop.operations, jump_values)
         if expected != "crash!":
             expected = self.parse(expected)
         if expected_preamble:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -233,6 +233,8 @@
     simplevalue = cpu.fielddescrof(SIMPLE, 'value')
     simple_vtable = lltype.malloc(OBJECT_VTABLE, immortal=True)
     simpledescr = cpu.sizeof(SIMPLE, simple_vtable)
+    simple = lltype.malloc(SIMPLE, zero=True)
+    simpleaddr = lltype.cast_opaque_ptr(llmemory.GCREF, simple)
     #usize = cpu.sizeof(U, ...)
     onedescr = cpu.fielddescrof(U, 'one')
 
@@ -515,6 +517,19 @@
         return Info(preamble, loop_info.target_token.short_preamble,
                     start_state.virtual_state)
 
+    def set_values(self, ops, jump_values=None):
+        jump_op = ops[-1]
+        assert jump_op.getopnum() == rop.JUMP
+        if jump_values is not None:
+            for i, v in enumerate(jump_values):
+                if v is not None:
+                    jump_op.getarg(i).setref_base(v)
+        else:
+            for i, box in enumerate(jump_op.getarglist()):
+                if box.type == 'r' and not box.is_constant():
+                    box.setref_base(self.nodefulladdr)
+
+
 
 class FakeDescr(compile.ResumeGuardDescr):
     def clone_if_mutable(self):

From noreply at buildbot.pypy.org  Thu Aug 27 12:32:05 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 12:32:05 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix 2 tests
Message-ID: <20150827103205.2E0DA1C05CE@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79249:3b1e02116b5d
Date: 2015-08-27 12:32 +0200
http://bitbucket.org/pypy/pypy/changeset/3b1e02116b5d/

Log:	fix 2 tests

diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -3447,7 +3447,7 @@
             return s
 
         self.meta_interp(f, [1, 10], inline=True)
-        self.check_resops(call=0, call_may_force=0, call_assembler=2)
+        self.check_resops(call_i=0, call_may_force_i=0, call_assembler_i=2)
 
     def test_reuse_elidable_result(self):
         driver = JitDriver(reds=['n', 's'], greens = [])
@@ -3461,7 +3461,7 @@
         res = self.meta_interp(main, [10])
         assert res == main(10)
         self.check_resops({'int_gt': 2, 'strlen': 2, 'guard_true': 2,
-                           'int_sub': 2, 'jump': 1, 'call': 2,
+                           'int_sub': 2, 'jump': 1, 'call_r': 2,
                            'guard_no_exception': 2, 'int_add': 4})
 
     def test_elidable_method(self):
@@ -3514,10 +3514,10 @@
         res = self.meta_interp(main, [0, 10])
         assert res == main(0, 10)
         # 2 calls, one for f() and one for char_mul
-        self.check_resops(call=4)
+        self.check_resops(call_i=2, call_r=2)
         res = self.meta_interp(main, [1, 10])
         assert res == main(1, 10)
-        self.check_resops(call=0)
+        self.check_resops(call_i=0, call_r=0)
 
     def test_setarrayitem_followed_by_arraycopy(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'sa', 'x', 'y'])

From noreply at buildbot.pypy.org  Thu Aug 27 12:39:39 2015
From: noreply at buildbot.pypy.org (cfbolz)
Date: Thu, 27 Aug 2015 12:39:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: reinstante accidentally killed
	line
Message-ID: <20150827103939.C5A561C0206@cobra.cs.uni-duesseldorf.de>

Author: Carl Friedrich Bolz 
Branch: optresult-unroll
Changeset: r79250:d06c03e5917f
Date: 2015-08-27 12:39 +0200
http://bitbucket.org/pypy/pypy/changeset/d06c03e5917f/

Log:	reinstante accidentally killed line

diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -726,6 +726,7 @@
             else:
                 assert type == 'f'
                 assert resvalue == upd.currfieldbox.getfloatstorage()
+            return upd.currfieldbox
         resbox = self.execute_with_descr(opnum, fielddescr, box)
         upd.getfield_now_known(resbox)
         return resbox

From noreply at buildbot.pypy.org  Thu Aug 27 13:01:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 13:01:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix
Message-ID: <20150827110111.BA4DE1C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79251:179bcf4602f2
Date: 2015-08-27 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/179bcf4602f2/

Log:	fix

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -52,11 +52,11 @@
                 self.make_constant_class(op, known_class, False)
             if isinstance(preamble_info, info.ArrayPtrInfo):
                 arr_info = info.ArrayPtrInfo(preamble_info.arraydescr)
-                arr_info.lenbound = preamble_info.getlenbound(None)
+                arr_info.lenbound = preamble_info.getlenbound(None).clone()
                 op.set_forwarded(arr_info)
             if isinstance(preamble_info, StrPtrInfo):
                 str_info = StrPtrInfo(preamble_info.mode)
-                str_info.lenbound = preamble_info.getlenbound(None)
+                str_info.lenbound = preamble_info.getlenbound(None).clone()
                 op.set_forwarded(str_info)
             if preamble_info.is_nonnull():
                 self.make_nonnull(op)
diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py
--- a/rpython/jit/metainterp/test/test_string.py
+++ b/rpython/jit/metainterp/test/test_string.py
@@ -147,7 +147,7 @@
         self.meta_interp(f, [6, 7])
         self.check_resops(newstr=0, strsetitem=0,
                          newunicode=0, unicodesetitem=0,
-                         call=0, call_pure=0)
+                         call_i=0, call_pure_i=0, call_r=0, call_pure_r=0)
 
     def test_strconcat_escape_str_str(self):
         _str = self._str
@@ -870,8 +870,8 @@
 
 
 class TestLLtype(StringTests, LLJitMixin):
-    CALL = "call"
-    CALL_PURE = "call_pure"
+    CALL = "call_i"
+    CALL_PURE = "call_pure_i"
 
 
 class TestLLtypeUnicode(TestLLtype):

From noreply at buildbot.pypy.org  Thu Aug 27 13:10:42 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 27 Aug 2015 13:10:42 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: extending test cases for memory
	references
Message-ID: <20150827111042.08FFC1C05CE@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79252:fc31b6dcacc8
Date: 2015-08-27 13:10 +0200
http://bitbucket.org/pypy/pypy/changeset/fc31b6dcacc8/

Log:	extending test cases for memory references

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -934,11 +934,11 @@
     """ IndexVar is an AbstractValue only to ensure that a box can be assigned
         to the same variable as an index var.
     """
-    def __init__(self, var):
+    def __init__(self, var, coeff_mul=1, coeff_div=1, constant=0):
         self.var = var
-        self.coefficient_mul = 1
-        self.coefficient_div = 1
-        self.constant = 0
+        self.coefficient_mul = coeff_mul
+        self.coefficient_div = coeff_div
+        self.constant = constant
         # saves the next modification that uses a variable
         self.next_nonconst = None
         self.current_end = None
@@ -990,7 +990,7 @@
 
     def same_variable(self, other):
         assert isinstance(other, IndexVar)
-        return other.var == self.var
+        return other.var is self.var
 
     def diff(self, other):
         """ calculates the difference as a second parameter """
@@ -1060,9 +1060,8 @@
         if self.is_identity():
             return 'IndexVar(%s+%s)' % (self.var, repr(self.next_nonconst))
 
-        return 'IndexVar((%s*(%s/%s)+%s) + %s)' % (self.var, self.coefficient_mul,
-                                            self.coefficient_div, self.constant,
-                                            repr(self.next_nonconst))
+        return 'IndexVar((%s*(%s/%s)+%s))' % (self.var, self.coefficient_mul,
+                                            self.coefficient_div, self.constant)
 
 class MemoryRef(object):
     """ a memory reference to an array object. IntegralForwardModification is able
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -7,20 +7,11 @@
 from rpython.jit.metainterp.optimizeopt.dependency import (DependencyGraph, Dependency,
         IndexVar, MemoryRef, Node)
 from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.backend.llgraph.runner import ArrayDescr
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype
 from rpython.conftest import option
 
-class FakeNode(Node):
-    def __init__(self, i):
-        Node.__init__(self, None, i)
-        pass
-
-    def __repr__(self):
-        return "n%d" % self.opidx
-
-class FakeDependencyGraph(DependencyGraph):
-    def __init__(self):
-        pass
-
 class DependencyBaseTest(BaseTest):
 
     def setup_method(self, method):
@@ -138,6 +129,48 @@
         return self.last_graph.memory_refs[node]
 
 class BaseTestDependencyGraph(DependencyBaseTest):
+
+    def test_index_var_basic(self):
+        b = FakeBox()
+        i = IndexVar(b,1,1,0)
+        j = IndexVar(b,1,1,0)
+        assert i.is_identity()
+        assert not i.less(j)
+        assert i.same_variable(j)
+        assert i.diff(j) == 0
+
+    def test_index_var_diff(self):
+        b = FakeBox()
+        i = IndexVar(b,4,2,0)
+        j = IndexVar(b,1,1,1)
+        assert not i.is_identity()
+        assert not j.is_identity()
+        assert i.diff(j) == 1
+
+    def test_memoryref_basic(self):
+        i = FakeBox()
+        a = FakeBox()
+        m1 = memoryref(a, i, (1,1,0))
+        m2 = memoryref(a, i, (1,1,0))
+        assert m1.match(m2)
+
+    @py.test.mark.parametrize('coeff1,coeff2,adja,alias',
+            [((1,1,0), (1,1,0), False, True),
+             ((4,2,0), (8,4,0), False, True),
+             ((4,2,0), (8,2,0), False, True),
+             ((4,2,1), (8,4,0), True, False),
+            ])
+    def test_memoryref_adjacent_alias(self, coeff1,
+                                      coeff2, adja,
+                                      alias):
+        i = FakeBox()
+        a = FakeBox()
+        m1 = memoryref(a, i, coeff1)
+        m2 = memoryref(a, i, coeff2)
+        assert m1.is_adjacent_after(m2) == adja
+        assert m2.is_adjacent_after(m1) == adja
+        assert m1.indices_can_alias(m2) == alias
+
     def test_dependency_empty(self):
         ops = """
         [] # 0: 1
@@ -511,6 +544,49 @@
         assert cycle is not None
         assert cycle == [n1,n3,n4]
 
+class FakeMemoryRefResOp(object):
+    def __init__(self, array, descr):
+        self.array = array
+        self.descr = descr
+    def getarg(self, index):
+        return self.array
+    def getdescr(self):
+        return self.descr
+
+FLOAT = ArrayDescr(lltype.Float)
+SFLOAT = ArrayDescr(lltype.SingleFloat)
+CHAR = ArrayDescr(rffi.r_signedchar)
+UCHAR = ArrayDescr(rffi.r_uchar)
+SHORT = ArrayDescr(rffi.r_short)
+USHORT = ArrayDescr(rffi.r_ushort)
+INT = ArrayDescr(rffi.r_int)
+UINT = ArrayDescr(rffi.r_uint)
+LONG = ArrayDescr(rffi.r_longlong)
+ULONG = ArrayDescr(rffi.r_ulonglong)
+
+def memoryref(array, var, mod=(1,1,0), descr=None, raw=False):
+    if descr is None:
+        descr = FLOAT
+    mul, div, off = mod
+    op = FakeMemoryRefResOp(array, descr)
+    return MemoryRef(op,
+                     IndexVar(var, mul, div, off),
+                     raw)
+
+class FakeBox(object):
+    pass
+
+class FakeNode(Node):
+    def __init__(self, i):
+        Node.__init__(self, None, i)
+        pass
+
+    def __repr__(self):
+        return "n%d" % self.opidx
+
+class FakeDependencyGraph(DependencyGraph):
+    def __init__(self):
+        pass
 
 
 class TestLLtype(BaseTestDependencyGraph, LLtypeMixin):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -326,6 +326,8 @@
                     if memref_a.is_adjacent_to(memref_b):
                         pair = self.packset.can_be_packed(node_a, node_b, None, False)
                         if pair:
+                            if node_a.op.getopnum() == rop.GETARRAYITEM_RAW:
+                                print "found", memref_a.index_var, memref_b.index_var
                             self.packset.add_pack(pair)
 
     def extend_packset(self):
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -13,6 +13,11 @@
 from rpython.rlib.rawstorage import (alloc_raw_storage, raw_storage_setitem,
                                      free_raw_storage, raw_storage_getitem)
 
+def malloc(T,n):
+    return lltype.malloc(T, n, flavor='raw', zero=True)
+def free(mem):
+    lltype.free(mem, flavor='raw')
+
 class VectorizeTests:
     enable_opts = 'intbounds:rewrite:virtualize:string:earlyforce:pure:heap:unroll'
 
@@ -289,8 +294,8 @@
         myjitdriver = JitDriver(greens = [],
                                 reds = 'auto')
         def f(d, v1, v2):
-            a = [v1] * d
-            b = [v2] * d
+            a = [v1] * i
+            b = [v2] * i
             i = 0
             while i < len(a):
                 myjitdriver.jit_merge_point()
@@ -305,6 +310,31 @@
         # return 69.999 ... instead of 70, (v1+v2)*i == 70.0
         assert res == f(i,v1,v2) == sum([v1+v2]*i)
 
+    @py.test.mark.parametrize('size',[12])
+    def test_body_multiple_accesses(self, size):
+        myjitdriver = JitDriver(greens = [], reds = 'auto')
+        T = lltype.Array(rffi.CHAR, hints={'nolength': True})
+        def f(size):
+            vector_a = malloc(T, size)
+            vector_b = malloc(T, size)
+            i = 0
+            while i < size:
+                myjitdriver.jit_merge_point()
+                # should unroll and group them correctly
+                c1 = vector_a[i]
+                c2 = vector_a[i+1]
+                c3 = vector_a[i+2]
+                #
+                vector_b[i] = c1
+                vector_b[i+1] = c2
+                vector_b[i+2] = c3
+                i += 3
+            free(vector_a)
+            free(vector_b)
+            return 0
+        res = self.meta_interp(f, [size], vec_all=True)
+        assert res == f(size)
+
 class VectorizeLLtypeTests(VectorizeTests):
     pass
 

From noreply at buildbot.pypy.org  Thu Aug 27 13:54:51 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 13:54:51 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: yay pass test_string
Message-ID: <20150827115451.91AC61C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79253:1e0e6b76cf77
Date: 2015-08-27 13:54 +0200
http://bitbucket.org/pypy/pypy/changeset/1e0e6b76cf77/

Log:	yay pass test_string

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -764,6 +764,12 @@
 
     def check_resops(self, expected=None, omit_finish=True, **check):
         insns = {}
+        if 'call' in check:
+            assert check.pop('call') == 0
+            check['call_i'] = check['call_r'] = check['call_f'] = check['call_n'] = 0
+        if 'call_pure' in check:
+            assert check.pop('call_pure') == 0
+            check['call_pure_i'] = check['call_pure_r'] = check['call_pure_f'] = 0
         for loop in self.get_all_loops():
             insns = loop.summary(adding_insns=insns, omit_finish=omit_finish)
         return self._check_insns(insns, expected, check)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -149,6 +149,11 @@
         cell_token = jump_op.getdescr()
         if not inline_short_preamble or len(cell_token.target_tokens) == 1:
             return self.jump_to_preamble(cell_token, jump_op, info)
+        # force all the information that does not go to the short
+        # preamble at all
+        self.optimizer.force_at_end_of_preamble()
+        for a in jump_op.getarglist():
+            self.optimizer.force_box_for_end_of_preamble(a)
         vs = self.jump_to_existing_trace(jump_op)
         if vs is None:
             return info, self.optimizer._newoperations[:]
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -653,4 +653,3 @@
 
     def visit_varraystruct(self, arraydescr, length, fielddescrs):
         return VArrayStructStateInfo(arraydescr, fielddescrs)
-
diff --git a/rpython/jit/metainterp/test/test_string.py b/rpython/jit/metainterp/test/test_string.py
--- a/rpython/jit/metainterp/test/test_string.py
+++ b/rpython/jit/metainterp/test/test_string.py
@@ -166,10 +166,11 @@
             return 42
         self.meta_interp(f, [6, 7])
         if _str is str:
-            self.check_resops(call_pure=0, copystrcontent=4,
-                              strsetitem=0, call=2, newstr=2)
+            self.check_resops(call_pure_i=0, call_pure_r=0, copystrcontent=4,
+                              strsetitem=0, call_n=2, newstr=2)
         else:
-            self.check_resops(call_pure=0, unicodesetitem=0, call=2,
+            self.check_resops(call_pure_i=0, call_pure_r=0,
+                              unicodesetitem=0, call_n=2,
                               copyunicodecontent=4, newunicode=2)
 
     def test_strconcat_escape_str_char(self):
@@ -189,10 +190,12 @@
             return 42
         self.meta_interp(f, [6, 7])
         if _str is str:
-            self.check_resops(call_pure=0, copystrcontent=2, strsetitem=2,
-                              call=2, newstr=2)
+            self.check_resops(call_pure_i=0, call_pure_r=0,
+                              copystrcontent=2, strsetitem=2,
+                              call_n=2, newstr=2)
         else:
-            self.check_resops(call_pure=0, unicodesetitem=2, call=2,
+            self.check_resops(call_pure_i=0, call_pure_r=0,
+                              unicodesetitem=2, call_n=2,
                               copyunicodecontent=2, newunicode=2)
 
     def test_strconcat_escape_char_str(self):
@@ -212,10 +215,11 @@
             return 42
         self.meta_interp(f, [6, 7])
         if _str is str:
-            self.check_resops(call_pure=0, copystrcontent=2,
-                              strsetitem=2, call=2, newstr=2)
+            self.check_resops(call_pure_i=0, call_pure_r=0, copystrcontent=2,
+                              strsetitem=2, call_n=2, newstr=2)
         else:
-            self.check_resops(call_pure=0, unicodesetitem=2, call=2,
+            self.check_resops(call_pure_i=0, unicodesetitem=2,
+                              call_n=2,
                               copyunicodecontent=2, newunicode=2)
 
     def test_strconcat_escape_char_char(self):
@@ -235,9 +239,9 @@
         self.meta_interp(f, [6, 7])
         if _str is str:
             self.check_resops(call_pure=0, copystrcontent=0,
-                              strsetitem=4, call=2, newstr=2)
+                              strsetitem=4, call_n=2, newstr=2)
         else:
-            self.check_resops(call_pure=0, unicodesetitem=4, call=2,
+            self.check_resops(call_pure=0, unicodesetitem=4, call_n=2,
                               copyunicodecontent=0, newunicode=2)
 
     def test_strconcat_escape_str_char_str(self):
@@ -258,9 +262,9 @@
         self.meta_interp(f, [6, 7])
         if _str is str:
             self.check_resops(call_pure=0, copystrcontent=4, strsetitem=2,
-                              call=2, newstr=2)
+                              call_n=2, newstr=2)
         else:
-            self.check_resops(call_pure=0, unicodesetitem=2, call=2,
+            self.check_resops(call_pure=0, unicodesetitem=2, call_n=2,
                               copyunicodecontent=4, newunicode=2)
 
     def test_strconcat_guard_fail(self):
@@ -512,7 +516,7 @@
             return 0
 
         self.meta_interp(f, [0])
-        self.check_resops(call=7)
+        self.check_resops(call_r=2, call_i=5)
 
     def test_join_chars(self):
         jitdriver = JitDriver(reds=['a', 'b', 'c', 'i'], greens=[])
@@ -688,8 +692,9 @@
             return n
         res = self.meta_interp(f, [10], backendopt=True)
         assert res == 0
-        self.check_resops(call=6,    # (ll_append_res0, ll_append_0_2, ll_build)
-                                     # * 2 unroll
+        self.check_resops(call_n=4, call_r=2,
+        # (ll_append_res0, ll_append_0_2, ll_build)
+        # * 2 unroll
                           cond_call=0)
 
     def test_stringbuilder_append_len2_2(self):
@@ -710,7 +715,8 @@
             return n
         res = self.meta_interp(f, [10], backendopt=True)
         assert res == 0
-        self.check_resops(call=4,    # (ll_append_res0, ll_build) * 2 unroll
+        self.check_resops(call_n=2, call_r=2,
+        # (ll_append_res0, ll_build) * 2 unroll
                           cond_call=0)
 
     def test_stringbuilder_append_slice_1(self):
@@ -727,7 +733,7 @@
             return n
         res = self.meta_interp(f, [10], backendopt=True)
         assert res == 0
-        self.check_resops(call=6, cond_call=0,
+        self.check_resops(call_n=4, call_r=2, cond_call=0,
                           copyunicodecontent=0)
 
     def test_stringbuilder_append_slice_2(self):
@@ -785,7 +791,8 @@
             return n
         res = self.meta_interp(f, [10], backendopt=True)
         assert res == 0
-        self.check_resops(call=4)    # (_ll_append_multiple_char, build) * 2
+        self.check_resops(call_n=2, call_r=2)
+         # (_ll_append_multiple_char, build) * 2
 
     def test_stringbuilder_bug1(self):
         jitdriver = JitDriver(reds=['n', 's1'], greens=[])
@@ -901,7 +908,8 @@
         # loop is not safe to remove; see 56eebe9dd813.  We can still
         # remove the call to ll_str, because that cannot raise anything
         # else than MemoryError.
-        self.check_resops(unicodesetitem=2, newunicode=2, call=5,
+        self.check_resops(unicodesetitem=2, newunicode=2, call_n=2,
+                          call_r=3,
                           copyunicodecontent=2, unicodegetitem=0)
 
     def test_str2unicode_fold(self):
@@ -920,7 +928,7 @@
                 m -= 1
             return 42
         self.meta_interp(f, [6, 7])
-        self.check_resops(call_pure=0, unicodesetitem=0, call=2,
+        self.check_resops(call_pure=0, unicodesetitem=0, call_n=2,
                           newunicode=0, unicodegetitem=0,
                           copyunicodecontent=0)
 

From noreply at buildbot.pypy.org  Thu Aug 27 14:18:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 14:18:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: write support for varraystruct
	virtual info
Message-ID: <20150827121845.E240D1C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79254:ad90619d3fce
Date: 2015-08-27 14:18 +0200
http://bitbucket.org/pypy/pypy/changeset/ad90619d3fce/

Log:	write support for varraystruct virtual info

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -231,6 +231,9 @@
         self.arraydescr = runner.arraydescrof(A)
         self.fielddescr = runner.fielddescrof(A.OF, fieldname)
 
+    def get_index(self):
+        return self.fielddescr.get_index()
+
     def get_arraydescr(self):
         return self.arraydescr
 
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -320,6 +320,9 @@
         self.arraydescr = arraydescr
         self.fielddescr = fielddescr
 
+    def get_index(self):
+        return self.fielddescr.get_index()
+
     def get_arraydescr(self):
         return self.arraydescr
 
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,6 +1,7 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
-from rpython.jit.metainterp.optimizeopt import info
+from rpython.jit.metainterp.optimizeopt.info import ArrayPtrInfo,\
+     ArrayStructInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
@@ -53,6 +54,18 @@
         else:
             return InputArgInt(self.cpu.bh_getfield_gc_i(struct, descr))
 
+    def get_runtime_interiorfield(self, box, descr, i):
+        struct = box.getref_base()
+        if descr.is_pointer_field():
+            return InputArgRef(self.cpu.bh_getinteriorfield_gc_r(struct, i,
+                                                                 descr))
+        elif descr.is_float_field():
+            return InputArgFloat(self.cpu.bh_getinteriorfield_gc_f(struct, i,
+                                                                   descr))
+        else:
+            return InputArgInt(self.cpu.bh_getinteriorfield_gc_i(struct, i,
+                                                                 descr))
+
 class AbstractVirtualStateInfo(object):
     position = -1
 
@@ -92,7 +105,7 @@
                 'Generating guards for making the VirtualStates ' +
                 'at hand match have not been implemented')
 
-    def enum_forced_boxes(self, boxes, value, optimizer):
+    def enum_forced_boxes(self, boxes, structbox, optimizer, force_boxes=False):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -225,7 +238,7 @@
         for i in range(len(self.fieldstate)):
             if runtime_box is not None:
                 opinfo = state.optimizer.getptrinfo(box)
-                assert isinstance(opinfo, info.ArrayPtrInfo)
+                assert isinstance(opinfo, ArrayPtrInfo)
                 fieldbox = opinfo._items[i]
                 fieldbox_runtime = state.get_runtime_item(runtime_box,
                                             self.arraydescr, i)
@@ -257,12 +270,12 @@
 
 
 class VArrayStructStateInfo(AbstractVirtualStateInfo):
-    def __init__(self, arraydescr, fielddescrs):
+    def __init__(self, arraydescr, fielddescrs, length):
         self.arraydescr = arraydescr
         self.fielddescrs = fielddescrs
+        self.length = length
 
-    def _generate_guards(self, other, box, opinfo, state):
-        xxxx
+    def _generate_guards(self, other, box, runtime_box, state):
         if not isinstance(other, VArrayStructStateInfo):
             raise VirtualStatesCantMatch("other is not an VArrayStructStateInfo")
         if self.arraydescr is not other.arraydescr:
@@ -271,48 +284,54 @@
         if len(self.fielddescrs) != len(other.fielddescrs):
             raise VirtualStatesCantMatch("other has a different length")
 
-        p = 0
+        if len(self.fielddescrs) != len(other.fielddescrs):
+            raise VirtualStatesCantMatch("other has a different length")
+        for j, descr in enumerate(self.fielddescrs):
+            if descr is not other.fielddescrs[j]:
+                raise VirtualStatesCantMatch("other is a different kind of array")
         fieldbox = None
-        fieldinfo = None
-        for i in range(len(self.fielddescrs)):
-            if len(self.fielddescrs[i]) != len(other.fielddescrs[i]):
-                raise VirtualStatesCantMatch("other has a different length")
-            for j in range(len(self.fielddescrs[i])):
-                descr = self.fielddescrs[i][j]
-                if descr is not other.fielddescrs[i][j]:
-                    raise VirtualStatesCantMatch("other is a different kind of array")
+        fieldbox_runtime = None
+        opinfo = state.optimizer.getptrinfo(box)
+        for i in range(self.length):
+            for descr in self.fielddescrs:
+                index = i * len(self.fielddescrs) + descr.get_index()
+                fieldstate = self.fieldstate[index]
+                if fieldstate is None:
+                    continue
                 if box is not None:
-                    xxx
-                    assert isinstance(value, virtualize.VArrayStructValue)
-                    v = value._items[i][descr]
-                self.fieldstate[p].generate_guards(other.fieldstate[p],
-                                                   fieldbox, fieldinfo,
-                                                   state)
-                p += 1
+                    fieldbox = opinfo._items[index]
+                    fieldbox_runtime = state.get_runtime_interiorfield(
+                        runtime_box, descr, i)
+                self.fieldstate[index].generate_guards(other.fieldstate[index],
+                                       fieldbox, fieldbox_runtime, state)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
-            s.enum(virtual_state)
+            if s is not None:
+                s.enum(virtual_state)
 
-    def enum_forced_boxes(self, boxes, value, optimizer):
-        xxx
-        if not isinstance(value, virtualize.VArrayStructValue):
+    def enum_forced_boxes(self, boxes, structbox, optimizer, force_boxes=False):
+        opinfo = optimizer.getptrinfo(structbox)
+        if not isinstance(opinfo, ArrayStructInfo):
             raise BadVirtualState
-        if not value.is_virtual():
+        if not opinfo.is_virtual():
             raise BadVirtualState
-        if len(self.fielddescrs) > len(value._items):
-            raise BadVirtualState
-        p = 0
-        for i in range(len(self.fielddescrs)):
-            for j in range(len(self.fielddescrs[i])):
-                try:
-                    v = value._items[i][self.fielddescrs[i][j]]
-                except KeyError:
-                    raise BadVirtualState
-                s = self.fieldstate[p]
-                if s.position > self.position:
-                    s.enum_forced_boxes(boxes, v, optimizer)
-                p += 1
+        #if len(self.fielddescrs) > len(value._items):
+        #    raise BadVirtualState
+        for i in range(self.length):
+            for descr in self.fielddescrs:
+                index = i * len(self.fielddescrs) + descr.get_index()
+                fieldstate = self.fieldstate[index]
+                itembox = opinfo._items[i * len(self.fielddescrs) +
+                                        descr.get_index()]
+                if fieldstate is None:
+                    if itembox is not None:
+                        raise BadVirtualState
+                    continue
+                # I think itembox must be present here
+                if fieldstate.position > self.position:
+                    fieldstate.enum_forced_boxes(boxes, itembox, optimizer,
+                                                 force_boxes)
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStructStateInfo(%d):' % self.position)
@@ -652,4 +671,4 @@
         return VArrayStateInfo(arraydescr)
 
     def visit_varraystruct(self, arraydescr, length, fielddescrs):
-        return VArrayStructStateInfo(arraydescr, fielddescrs)
+        return VArrayStructStateInfo(arraydescr, fielddescrs, length)

From noreply at buildbot.pypy.org  Thu Aug 27 14:25:55 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 14:25:55 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests in
	test_recursive
Message-ID: <20150827122555.EDEC21C148B@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79255:033eed71b734
Date: 2015-08-27 14:25 +0200
http://bitbucket.org/pypy/pypy/changeset/033eed71b734/

Log:	fix some tests in test_recursive

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -770,6 +770,12 @@
         if 'call_pure' in check:
             assert check.pop('call_pure') == 0
             check['call_pure_i'] = check['call_pure_r'] = check['call_pure_f'] = 0
+        if 'call_may_force' in check:
+            assert check.pop('call_may_force') == 0
+            check['call_may_force_i'] = check['call_may_force_r'] = check['call_may_force_f'] = check['call_may_force_n'] = 0
+        if 'call_assembler' in check:
+            assert check.pop('call_assembler') == 0
+            check['call_assembler_i'] = check['call_assembler_r'] = check['call_assembler_f'] = check['call_assembler_n'] = 0
         for loop in self.get_all_loops():
             insns = loop.summary(adding_insns=insns, omit_finish=omit_finish)
         return self._check_insns(insns, expected, check)
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -144,10 +144,10 @@
         f = self.get_interpreter(codes)
 
         assert self.meta_interp(f, [0, 0, 0], enable_opts='') == 42
-        self.check_resops(call_may_force=1, int_add=1, call=0)
+        self.check_resops(call_may_force_i=1, int_add=1, call=0)
         assert self.meta_interp(f, [0, 0, 0], enable_opts='',
                                 inline=True) == 42
-        self.check_resops(call=0, int_add=2, call_may_force=0,
+        self.check_resops(call=0, int_add=2, call_may_force_i=0,
                           guard_no_exception=0)
 
     def test_inline_jitdriver_check(self):
@@ -490,7 +490,7 @@
             return loop(100)
 
         res = self.meta_interp(main, [0], enable_opts='', trace_limit=TRACE_LIMIT)
-        self.check_resops(call=0, call_may_force=1)
+        self.check_resops(call=0, call_may_force_i=1)
 
         res = self.meta_interp(main, [1], enable_opts='', trace_limit=TRACE_LIMIT)
         self.check_resops(call=0, call_may_force=0)
@@ -575,7 +575,7 @@
                 result += f('-c-----------l-', i+100)
         self.meta_interp(g, [10], backendopt=True)
         self.check_aborted_count(1)
-        self.check_resops(call=0, call_assembler=2)
+        self.check_resops(call=0, call_assembler_i=2)
         self.check_jitcell_token_count(2)
 
     def test_directly_call_assembler(self):
@@ -624,7 +624,7 @@
         try:
             compile.compile_tmp_callback = my_ctc
             self.meta_interp(portal, [2, 5], inline=True)
-            self.check_resops(call_may_force=0, call_assembler=2)
+            self.check_resops(call_may_force=0, call_assembler_n=2)
         finally:
             compile.compile_tmp_callback = original_ctc
         # check that we made a temporary callback
@@ -679,7 +679,7 @@
         try:
             compile.compile_tmp_callback = my_ctc
             self.meta_interp(main, [2, 5], inline=True)
-            self.check_resops(call_may_force=0, call_assembler=2)
+            self.check_resops(call_may_force=0, call_assembler_n=2)
         finally:
             compile.compile_tmp_callback = original_ctc
         # check that we made a temporary callback

From noreply at buildbot.pypy.org  Thu Aug 27 14:30:25 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 14:30:25 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150827123025.5CBA71C1FF4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79256:ca17931ca6a5
Date: 2015-08-27 14:30 +0200
http://bitbucket.org/pypy/pypy/changeset/ca17931ca6a5/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -33,7 +33,7 @@
         assert res == 55 * 10
         self.check_trace_count(1)
         self.check_resops(new_with_vtable=0, setfield_gc=0,
-                          getfield_gc=2, new=0)
+                          getfield_gc_i=2, new=0)
 
     def test_virtualized2(self):
         myjitdriver = JitDriver(greens=[], reds=['n', 'node1', 'node2'])
@@ -54,7 +54,7 @@
                 n -= 1
             return node1.value * node2.value
         assert f(10) == self.meta_interp(f, [10])
-        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=2,
+        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc_i=2,
                           new=0)
 
     def test_virtualized_circular1(self):
@@ -80,8 +80,8 @@
         res = self.meta_interp(f, [10])
         assert res == 55 * 10
         self.check_trace_count(1)
-        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=3,
-                          new=0)
+        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc_i=2,
+                          getfield_gc_r=1, new=0)
 
     def test_virtualized_float(self):
         myjitdriver = JitDriver(greens=[], reds=['n', 'node'])
@@ -140,7 +140,7 @@
         res = self.meta_interp(f, [10])
         assert res == 55 * 30
         self.check_trace_count(1)
-        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=2,
+        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc_i=2,
                           new=0)
 
     def test_nonvirtual_obj_delays_loop(self):
@@ -161,7 +161,7 @@
         res = self.meta_interp(f, [500])
         assert res == 640
         self.check_trace_count(1)
-        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=1,
+        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc_i=1,
                           new=0)
 
     def test_two_loops_with_virtual(self):
@@ -185,7 +185,7 @@
         res = self.meta_interp(f, [18])
         assert res == f(18)
         self.check_trace_count(2)
-        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=2,
+        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc_i=2,
                           new=0)
 
     def test_two_loops_with_escaping_virtual(self):
@@ -308,7 +308,7 @@
 
         res = self.meta_interp(f, [10, 1, 0], listops=True)
         assert res == 0
-        self.check_resops(getfield_gc=0)
+        self.check_resops(getfield_gc_i=0)
 
     def test_escapes(self):
         myjitdriver = JitDriver(greens=[], reds=['n', 'parent'])
@@ -1149,7 +1149,7 @@
 
         res = self.meta_interp(f, [16])
         assert res == f(16)
-        self.check_resops(getfield_gc=7)
+        self.check_resops(getfield_gc_i=7)
 
     def test_raw_malloc(self):
         mydriver = JitDriver(greens=[], reds='auto')
@@ -1214,7 +1214,7 @@
         assert res == 45
         # make sure that the raw buffer is *not* virtualized because we do not
         # support virtualstate
-        self.check_resops(getarrayitem_raw=2, raw_store=2)
+        self.check_resops(getarrayitem_raw_i=2, raw_store=2)
 
     def test_raw_malloc_only_chars(self):
         mydriver = JitDriver(greens=[], reds='auto')
@@ -1276,7 +1276,7 @@
         res = self.meta_interp(f, [10])
         assert res == 20
         self.check_trace_count(1)
-        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc=0,
+        self.check_resops(new_with_vtable=0, setfield_gc=0, getfield_gc_i=0,
                           new=0)
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Thu Aug 27 14:42:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 14:42:28 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fixes to the backend
Message-ID: <20150827124228.D8CA81C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79257:40407d03401e
Date: 2015-08-27 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/40407d03401e/

Log:	fixes to the backend

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -106,12 +106,17 @@
     field_size = 0
     flag = '\x00'
 
-    def __init__(self, name, offset, field_size, flag, index_in_parent=0):
+    def __init__(self, name, offset, field_size, flag, index_in_parent=0,
+                 is_pure=False):
         self.name = name
         self.offset = offset
         self.field_size = field_size
         self.flag = flag
         self.index = index_in_parent
+        self._is_pure = is_pure
+
+    def is_always_pure(self):
+        return self._is_pure
 
     def __repr__(self):
         return 'FieldDescr<%s>' % (self.name,)
@@ -184,7 +189,9 @@
         flag = get_type_flag(FIELDTYPE)
         name = '%s.%s' % (STRUCT._name, fieldname)
         index_in_parent = heaptracker.get_fielddescr_index_in(STRUCT, fieldname)
-        fielddescr = FieldDescr(name, offset, size, flag, index_in_parent)
+        is_pure = STRUCT._immutable_field(fieldname)
+        fielddescr = FieldDescr(name, offset, size, flag, index_in_parent,
+                                is_pure)
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         if STRUCT is rclass.OBJECT:
@@ -234,15 +241,19 @@
     vinfo = None
     all_interiorfielddescrs = None
 
-    def __init__(self, basesize, itemsize, lendescr, flag):
+    def __init__(self, basesize, itemsize, lendescr, flag, is_pure=False):
         self.basesize = basesize
         self.itemsize = itemsize
         self.lendescr = lendescr    # or None, if no length
         self.flag = flag
+        self._is_pure = is_pure
 
     def get_all_fielddescrs(self):
         return self.all_interiorfielddescrs
 
+    def is_always_pure(self):
+        return self._is_pure
+
     def is_array_of_pointers(self):
         return self.flag == FLAG_POINTER
 
@@ -297,7 +308,8 @@
         else:
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
         flag = get_type_flag(ARRAY_INSIDE.OF)
-        arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag)
+        is_pure = ARRAY_INSIDE._immutable_field(None)
+        arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag, is_pure)
         cache[ARRAY_OR_STRUCT] = arraydescr
         if isinstance(ARRAY_INSIDE.OF, lltype.Struct):
             descrs = heaptracker.all_interiorfielddescrs(gccache,
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -328,7 +328,7 @@
                                              loop_data)
     except InvalidLoop:
         # Fall back on jumping to preamble
-        xxx
+        raise Exception("think about it")
         return None
 
     loop = partial_trace

From noreply at buildbot.pypy.org  Thu Aug 27 15:16:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 15:16:58 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: work towards translation
Message-ID: <20150827131658.5491A1C0206@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79258:aea7066431e6
Date: 2015-08-27 15:16 +0200
http://bitbucket.org/pypy/pypy/changeset/aea7066431e6/

Log:	work towards translation

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -93,7 +93,7 @@
         assert not isinstance(vtable, bool)
         self.S = S
         self._vtable = vtable
-        self._is_object = vtable is not None
+        self._is_object = bool(vtable)
         self.all_fielddescrs = heaptracker.all_fielddescrs(runner, S,
                                     get_field_descr=LLGraphCPU.fielddescrof)
         self._runner = runner
diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -45,6 +45,7 @@
         self.gc_fielddescrs = gc_fielddescrs
         self.all_fielddescrs = all_fielddescrs
         self.vtable = vtable
+        assert vtable is not None
         self.immutable_flag = immutable_flag
 
     def get_all_fielddescrs(self):
@@ -195,7 +196,7 @@
         cachedict = cache.setdefault(STRUCT, {})
         cachedict[fieldname] = fielddescr
         if STRUCT is rclass.OBJECT:
-            vtable = None
+            vtable = lltype.nullptr(rclass.OBJECT_VTABLE)
         else:
             vtable = heaptracker.get_vtable_for_gcstruct(gccache, STRUCT)
         fielddescr.parent_descr = get_size_descr(gccache, STRUCT, vtable)
diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py
--- a/rpython/jit/codewriter/heaptracker.py
+++ b/rpython/jit/codewriter/heaptracker.py
@@ -41,9 +41,9 @@
     # lowleveltype, return the corresponding vtable pointer.
     # Returns None if the GcStruct does not belong to an instance.
     if not isinstance(GCSTRUCT, lltype.GcStruct):
-        return None
+        return lltype.nullptr(rclass.OBJECT_VTABLE)
     if not has_gcstruct_a_vtable(GCSTRUCT):
-        return None
+        return lltype.nullptr(rclass.OBJECT_VTABLE)
     setup_cache_gcstruct2vtable(gccache)
     try:
         return gccache._cache_gcstruct2vtable[GCSTRUCT]
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -935,7 +935,7 @@
             opname = 'new_with_vtable'
         else:
             opname = 'new'
-            vtable = None
+            vtable = lltype.nullptr(rclass.OBJECT_VTABLE)
         sizedescr = self.cpu.sizeof(STRUCT, vtable)
         op1 = SpaceOperation(opname, [sizedescr], op.result)
         if zero:
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -58,7 +58,7 @@
         else:
             opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
             return opt.propagate_all_forward(self.start_label.getarglist(),
-               self.operations, self.call_pure_results, self.enable_opts)
+               self.operations, self.call_pure_results)
 
 class SimpleCompileData(CompileData):
     """ This represents label() ops jump with no extra info associated with
@@ -77,7 +77,7 @@
         #assert not unroll
         opt = Optimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.propagate_all_forward(self.start_label.getarglist(),
-            self.operations, self.call_pure_results, self.enable_opts)
+            self.operations, self.call_pure_results)
 
 class BridgeCompileData(CompileData):
     """ This represents ops() with a jump at the end that goes to some
@@ -226,7 +226,8 @@
     label = ResOperation(rop.LABEL, loop_info.inputargs[:], descr=target_token)
     jump_op.setdescr(target_token)
     loop.operations = [label] + ops
-    loop.check_consistency()
+    if not we_are_translated():
+        loop.check_consistency()
     jitcell_token.target_tokens = [target_token]
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
@@ -289,10 +290,10 @@
                               original_jitcell_token=jitcell_token)
     start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs,
                                descr=start_descr)
-    # XXX assign short preamble and virtual state
     loop.operations = ([start_label] + preamble_ops + loop_info.extra_same_as +
                        [loop_info.label_op] + loop_ops)
-    loop.check_consistency()
+    if not we_are_translated():
+        loop.check_consistency()
     jitcell_token.target_tokens = [start_descr] + jitcell_token.target_tokens
     send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
     record_loop_or_bridge(metainterp_sd, loop)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -436,7 +436,8 @@
     def all_items(self):
         return self._items
 
-    def copy_fields_to_const(self, arraydescr, constinfo, optheap):
+    def copy_fields_to_const(self, constinfo, optheap):
+        arraydescr = self.arraydescr
         if self._items is not None:
             info = constinfo._get_array_info(arraydescr, optheap)
             info._items = self._items[:]
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,6 +1,6 @@
 
 import sys
-from rpython.jit.metainterp.history import Const, TargetToken
+from rpython.jit.metainterp.history import Const, TargetToken, JitCellToken
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
      ShortPreambleBuilder, PreambleOp
 from rpython.jit.metainterp.optimizeopt import info, intutils
@@ -98,7 +98,7 @@
     def optimize_preamble(self, start_label, end_label, ops, call_pure_results):
         self._check_no_forwarding([[start_label, end_label], ops])
         info, newops = self.optimizer.propagate_all_forward(
-            start_label.getarglist()[:], ops, call_pure_results)
+            start_label.getarglist()[:], ops, call_pure_results, True)
         exported_state = self.export_state(start_label, end_label.getarglist(),
                                            info.inputargs)
         # we need to absolutely make sure that we've cleaned up all
@@ -185,6 +185,7 @@
         self.optimizer._clean_optimization_info(sb.short_inputargs)
         short_preamble = sb.build_short_preamble()
         jitcelltoken = label_op.getdescr()
+        assert isinstance(jitcelltoken, JitCellToken)
         if jitcelltoken.target_tokens is None:
             jitcelltoken.target_tokens = []
         target_token = TargetToken(jitcelltoken,
@@ -205,6 +206,7 @@
 
     def jump_to_existing_trace(self, jump_op):
         jitcelltoken = jump_op.getdescr()
+        assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
         args = [self.get_box_replacement(op) for op in jump_op.getarglist()]
         for target_token in jitcelltoken.target_tokens:
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -152,7 +152,7 @@
             self._chars = [None] * length
         StrPtrInfo.__init__(self, mode, is_virtual, length)
 
-    def setitem(self, index, op, cf=None, optheap=None):
+    def strsetitem(self, index, op, cf=None, optheap=None):
         self._chars[index] = op
 
     def shrink(self, length):
@@ -165,7 +165,7 @@
         self._chars = longerlist[start:stop]
         # slice the 'longerlist', which may also contain Nones
 
-    def getitem(self, index, optheap=None):
+    def strgetitem(self, index, optheap=None):
         return self._chars[index]
 
     def is_virtual(self):
@@ -199,7 +199,7 @@
                                  offsetbox, mode):
         for i in range(len(self._chars)):
             assert not isinstance(targetbox, Const) # ConstPtr never makes sense
-            charbox = self.getitem(i) # can't be virtual
+            charbox = self.strgetitem(i) # can't be virtual
             if charbox is not None:
                 op = ResOperation(mode.STRSETITEM, [targetbox,
                                                     offsetbox,
@@ -452,7 +452,7 @@
         if opinfo and opinfo.is_virtual():
             indexbox = self.get_constant_box(op.getarg(1))
             if indexbox is not None:
-                opinfo.setitem(indexbox.getint(),
+                opinfo.strsetitem(indexbox.getint(),
                               self.get_box_replacement(op.getarg(2)))
                 return
         self.make_nonnull(op.getarg(0))
@@ -481,7 +481,7 @@
             # even if no longer virtual
             vindex = self.getintbound(index)
             if vindex.is_constant():
-                result = sinfo.getitem(vindex.getint())
+                result = sinfo.strgetitem(vindex.getint())
                 if result is not None:
                     if op is not None:
                         self.make_equal_to(op, result)
@@ -549,7 +549,7 @@
                 vresult = self.strgetitem(None, op.getarg(0),
                                           ConstInt(index + src_start), mode)
                 if dst_virtual:
-                    dst.setitem(index + dst_start, vresult)
+                    dst.strsetitem(index + dst_start, vresult)
                 else:
                     new_op = ResOperation(mode.STRSETITEM, [
                         op.getarg(1), ConstInt(index + dst_start),

From noreply at buildbot.pypy.org  Thu Aug 27 16:34:35 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 27 Aug 2015 16:34:35 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: loop versioning needed to changes
 the failargs for stitched bridges -> thus tests needed adaption
Message-ID: <20150827143435.6C9531C0695@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79259:d95ae9b7cad9
Date: 2015-08-27 16:34 +0200
http://bitbucket.org/pypy/pypy/changeset/d95ae9b7cad9/

Log:	loop versioning needed to changes the failargs for stitched bridges
	-> thus tests needed adaption further testing memory ref
	adjacent_to, adjacent_after and alias work correctly

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -449,7 +449,7 @@
                     while i >= 0:
                         def_node = def_chain[i][0]
                         oref = def_node.memory_ref
-                        if oref is not None and mref.indices_can_alias(oref):
+                        if oref is not None and mref.alias(oref):
                             return def_node
                         elif oref is None:
                             return def_node
@@ -968,17 +968,10 @@
                self.coefficient_div == 1 and \
                self.constant == 0
 
-    def __eq__(self, other):
+    def less(self, other):
+        # TODO
         if self.same_variable(other):
-            return self.diff(other) == 0
-        return False
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def less(self, other):
-        if self.same_variable(other):
-            return self.diff(other) < 0
+            return self.constant_diff(other) < 0
         return False
 
     def clone(self):
@@ -990,14 +983,26 @@
 
     def same_variable(self, other):
         assert isinstance(other, IndexVar)
-        return other.var is self.var
+        return other.var == self.var
 
-    def diff(self, other):
+    def same_mulfactor(self, other):
+        coeff = self.coefficient_mul == other.coefficient_mul
+        coeff = coeff and (self.coefficient_div == other.coefficient_div)
+        if not coeff:
+            # if not equal, try to check if they divide without rest
+            selfmod = self.coefficient_mul % self.coefficient_div
+            othermod = other.coefficient_mul % other.coefficient_div
+            if selfmod == 0 and othermod == 0:
+                # yet another chance for them to be equal
+                selfdiv = self.coefficient_mul // self.coefficient_div
+                otherdiv = other.coefficient_mul // other.coefficient_div
+                coeff = selfdiv == otherdiv
+        return coeff
+
+    def constant_diff(self, other):
         """ calculates the difference as a second parameter """
         assert isinstance(other, IndexVar)
-        mycoeff = self.coefficient_mul // self.coefficient_div
-        othercoeff = other.coefficient_mul // other.coefficient_div
-        return mycoeff + self.constant - (othercoeff + other.constant)
+        return self.constant - other.constant
 
     def emit_operations(self, opt, result_box=None):
         box = self.var
@@ -1029,7 +1034,7 @@
         return box
 
     def compare(self, other):
-        """ returns if the two are compareable as a first result
+        """ Returns if the two are compareable as a first result
             and a number (-1,0,1) of the ordering
         """
         coeff = self.coefficient_mul == other.coefficient_mul
@@ -1056,6 +1061,16 @@
             return True, c
         return False, 0
 
+    def __eq__(self, other):
+        if not self.same_variable(other):
+            return False
+        if not self.same_mulfactor(other):
+            return False
+        return self.constant_diff(other) == 0
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
     def __repr__(self):
         if self.is_identity():
             return 'IndexVar(%s+%s)' % (self.var, repr(self.next_nonconst))
@@ -1083,16 +1098,58 @@
 
     def is_adjacent_to(self, other):
         """ this is a symmetric relation """
+        if not self.same_array(other):
+            return False
+        if not self.index_var.same_variable(other.index_var):
+            return False
+        if not self.index_var.same_mulfactor(other.index_var):
+            return False
         stride = self.stride()
-        if self.match(other):
-            return abs(self.index_var.diff(other.index_var)) - stride == 0
-        return False
+        return abs(self.index_var.constant_diff(other.index_var)) - stride == 0
 
-    def match(self, other):
-        assert isinstance(other, MemoryRef)
-        if self.array == other.array and self.descr == other.descr:
-            return self.index_var.same_variable(other.index_var)
-        return False
+    def is_adjacent_after(self, other):
+        """ the asymetric relation to is_adjacent_to """
+        if not self.same_array(other):
+            return False
+        if not self.index_var.same_variable(other.index_var):
+            return False
+        if not self.index_var.same_mulfactor(other.index_var):
+            return False
+        stride = self.stride()
+        return other.index_var.constant_diff(self.index_var) == stride
+
+    def alias(self, other):
+        """ is this reference an alias to other?
+            they can alias iff self.origin != other.origin, or their
+            linear combination point to the same element.
+        """
+        assert other is not None
+        if not self.same_array(other):
+            return False
+        svar = self.index_var
+        ovar = other.index_var
+        if not svar.same_variable(ovar):
+            return True
+        if not svar.same_mulfactor(ovar):
+            return True
+        return abs(svar.constant_diff(ovar)) < self.stride()
+
+    def same_array(self, other):
+        return self.array is other.array and self.descr == other.descr
+
+    def __eq__(self, other):
+        """ NOT_RPYTHON """
+        if not self.same_array(other):
+            return False
+        if not self.index_var.same_variable(other.index_var):
+            return False
+        if not self.index_var.same_mulfactor(other.index_var):
+            return False
+        stride = self.stride()
+        return other.index_var.constant_diff(self.index_var) == 0
+
+    #def __ne__(self, other):
+    #    return not self.__eq__(other)
 
     def stride(self):
         """ the stride in bytes """
@@ -1100,34 +1157,5 @@
             return 1
         return self.descr.get_item_size_in_bytes()
 
-    def is_adjacent_after(self, other):
-        """ the asymetric relation to is_adjacent_to """
-        stride = self.stride()
-        if self.match(other):
-            return other.index_var.diff(self.index_var) == stride
-        return False
-
-    def indices_can_alias(self, other):
-        """ can to array indices alias? they can alias iff 
-        self.origin != other.origin, or their
-        linear combination point to the same element.
-        """
-        assert other is not None
-        if not self.index_var.same_variable(other.index_var):
-            return True
-        stride = self.stride()
-        if self.match(other):
-            diff = self.index_var.diff(other.index_var)
-            return abs(diff) < stride
-        return False
-
-    def __eq__(self, other):
-        if self.match(other):
-            return self.index_var.diff(other.index_var) == 0
-        return False
-
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
     def __repr__(self):
         return 'MemRef(%s,%s)' % (self.array, self.index_var)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_dependency.py
@@ -137,7 +137,7 @@
         assert i.is_identity()
         assert not i.less(j)
         assert i.same_variable(j)
-        assert i.diff(j) == 0
+        assert i.constant_diff(j) == 0
 
     def test_index_var_diff(self):
         b = FakeBox()
@@ -145,31 +145,41 @@
         j = IndexVar(b,1,1,1)
         assert not i.is_identity()
         assert not j.is_identity()
-        assert i.diff(j) == 1
+        assert not i.same_mulfactor(j)
+        assert i.constant_diff(j) == -1
 
     def test_memoryref_basic(self):
         i = FakeBox()
         a = FakeBox()
         m1 = memoryref(a, i, (1,1,0))
         m2 = memoryref(a, i, (1,1,0))
-        assert m1.match(m2)
+        assert m1.alias(m2)
 
-    @py.test.mark.parametrize('coeff1,coeff2,adja,alias',
-            [((1,1,0), (1,1,0), False, True),
-             ((4,2,0), (8,4,0), False, True),
-             ((4,2,0), (8,2,0), False, True),
-             ((4,2,1), (8,4,0), True, False),
+    @py.test.mark.parametrize('coeff1,coeff2,state',
+            #                    +------------------ adjacent
+            #                    |+----------------- adjacent_after
+            #                    ||+---------------- adjacent_befure
+            #                    |||+--------------- alias
+            #                    ||||
+            [((1,1,0), (1,1,0), 'ffft'),
+             ((4,2,0), (8,4,0), 'ffft'),
+             ((4,2,0), (8,2,0), 'ffft'),
+             ((4,2,1), (8,4,0), 'tftf'),
             ])
-    def test_memoryref_adjacent_alias(self, coeff1,
-                                      coeff2, adja,
-                                      alias):
+    def test_memoryref_adjacent_alias(self, coeff1, coeff2, state):
         i = FakeBox()
         a = FakeBox()
         m1 = memoryref(a, i, coeff1)
         m2 = memoryref(a, i, coeff2)
-        assert m1.is_adjacent_after(m2) == adja
-        assert m2.is_adjacent_after(m1) == adja
-        assert m1.indices_can_alias(m2) == alias
+        adja = state[0] == 't'
+        adja_after = state[1] == 't'
+        adja_before = state[2] == 't'
+        alias = state[3] == 't'
+        assert m1.is_adjacent_to(m2) == adja
+        assert m2.is_adjacent_to(m1) == adja
+        assert m1.is_adjacent_after(m2) == adja_after
+        assert m2.is_adjacent_after(m1) == adja_before
+        assert m1.alias(m2) == alias
 
     def test_dependency_empty(self):
         ops = """
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_guard.py b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_guard.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_guard.py
@@ -80,6 +80,7 @@
 
 class GuardBaseTest(SchedulerBaseTest):
     def optguards(self, loop, user_code=False):
+        loop.snapshot()
         for op in loop.operations:
             if op.is_guard():
                 op.setdescr(compile.CompileLoopVersionDescr())
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -76,6 +76,7 @@
         return opt
 
     def vectoroptimizer_unrolled(self, loop, unroll_factor = -1):
+        loop.snapshot()
         opt = self.vectoroptimizer(loop)
         opt.linear_find_smallest_type(loop)
         if unroll_factor == -1 and opt.smallest_type_bytes == 0:
@@ -1024,7 +1025,7 @@
         i30 = int_lt(i20, 10)
         i2 = int_add(i0, 2)
         i3 = int_lt(i2, 10)
-        guard_true(i3) [p0,i0]
+        guard_true(i3) [p0,i0,f3]
         i4 = int_add(i0, 2)
         i5 = int_lt(i2, 10)
         v1 = vec_getarrayitem_raw(p0, i0, 2, descr=floatarraydescr)
@@ -1090,7 +1091,7 @@
         i52 = int_ge(i50, i18) 
         i637 = int_add(i28, 2)
         i638 = int_ge(i637, i18)
-        guard_false(i638) [p38, p12, p9, p14, p39, i37, i44, f35, i40, p42, i43, f34, i28, p36, i41]
+        guard_false(i638) [p36, i28, p9, i37, p14, f34, p12, p38, f35, p39, i40, i41, p42, i43, i44, i21, i4, i0, i18]
         i55 = int_add(i44, 16) 
         i629 = int_add(i28, 2)
         i57 = int_ge(i637, i18) 
@@ -1123,7 +1124,7 @@
         i4 = int_ge(i3, 36)
         i50 = int_add(i1, 4)
         i51 = int_ge(i50, 36)
-        guard_false(i51) []
+        guard_false(i51) [p0, p1, i1]
         i5 = int_add(i1, 2)
         i8 = int_ge(i5, 36)
         i6 = int_add(i1, 3)
@@ -1165,7 +1166,7 @@
         i186 = int_lt(i5, 100)
         i500 = int_add(i4, 16)
         i501 = int_lt(i500, 100)
-        guard_true(i501) []
+        guard_true(i501) [p0, p1, p2, i0, i4]
         i189 = int_add(i0, 8)
         i187 = int_add(i4, 8)
         i198 = int_add(i0, 12)

From noreply at buildbot.pypy.org  Thu Aug 27 18:39:36 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 18:39:36 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: work on rpythonization
Message-ID: <20150827163937.00A301C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79260:1e42367301da
Date: 2015-08-27 18:39 +0200
http://bitbucket.org/pypy/pypy/changeset/1e42367301da/

Log:	work on rpythonization

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -523,6 +523,7 @@
 
     @staticmethod
     def check_consistency_of(inputargs, operations, check_descr=True):
+        "NOT_RPYTHON"
         for box in inputargs:
             assert not isinstance(box, Const), "Loop.inputargs contains %r" % (box,)
         seen = dict.fromkeys(inputargs)
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -67,7 +67,7 @@
         pass
 
     def make_guards(self, op, short):
-        short.append(ResOperation(rop.GUARD_VALUE, [op, self._const]))
+        pass
     
 class NonNullPtrInfo(PtrInfo):
     _attrs_ = ('last_guard_pos',)
@@ -158,6 +158,7 @@
     def copy_fields_to_const(self, constinfo, optheap):
         if self._fields is not None:
             info = constinfo._get_info(optheap)
+            assert isinstance(info, AbstractStructPtrInfo)
             info._fields = self._fields[:]
 
     def all_items(self):
@@ -440,6 +441,7 @@
         arraydescr = self.arraydescr
         if self._items is not None:
             info = constinfo._get_array_info(arraydescr, optheap)
+            assert isinstance(info, ArrayPtrInfo)
             info._items = self._items[:]
 
     def _force_elements(self, op, optforce, descr):
@@ -599,6 +601,9 @@
     def getconst(self):
         return self._const
 
+    def make_guards(self, op, short):
+        short.append(ResOperation(rop.GUARD_VALUE, [op, self._const]))
+
     def _get_info(self, optheap):
         ref = self._const.getref_base()
         info = optheap.const_infos.get(ref, None)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -1,5 +1,6 @@
 
 from collections import OrderedDict
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.resoperation import ResOperation, OpHelpers,\
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
@@ -409,6 +410,7 @@
         label_op = ResOperation(rop.LABEL, self.short_inputargs[:])
         jump_op = ResOperation(rop.JUMP, self.short_preamble_jump)
         # WARNING! the short_preamble_jump is shared on purpose
-        TreeLoop.check_consistency_of(self.short_inputargs,
-                                      self.short + [jump_op], check_descr=False)
+        if not we_are_translated():
+            TreeLoop.check_consistency_of(self.short_inputargs,
+                                self.short + [jump_op], check_descr=False)
         return [label_op] + self.short + [jump_op]
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -52,11 +52,15 @@
                 self.make_constant_class(op, known_class, False)
             if isinstance(preamble_info, info.ArrayPtrInfo):
                 arr_info = info.ArrayPtrInfo(preamble_info.arraydescr)
-                arr_info.lenbound = preamble_info.getlenbound(None).clone()
+                bound = preamble_info.getlenbound(None).clone()
+                assert isinstance(bound, intutils.IntBound)
+                arr_info.lenbound = bound
                 op.set_forwarded(arr_info)
             if isinstance(preamble_info, StrPtrInfo):
                 str_info = StrPtrInfo(preamble_info.mode)
-                str_info.lenbound = preamble_info.getlenbound(None).clone()
+                bound = preamble_info.getlenbound(None).clone()
+                assert isinstance(bound, intutils.IntBound)
+                str_info.lenbound = bound
                 op.set_forwarded(str_info)
             if preamble_info.is_nonnull():
                 self.make_nonnull(op)
@@ -130,7 +134,9 @@
         # the label_op again
         label_op.initarglist(label_args + extra)
         if new_virtual_state is not None:
-            self.jump_to_preamble(start_label.getdescr(), end_jump, info)
+            celltoken = start_label.getdescr()
+            assert isinstance(celltoken, JitCellToken)
+            self.jump_to_preamble(celltoken, end_jump, info)
             return (UnrollInfo(target_token, label_op, []),
                     self.optimizer._newoperations)
             #return new_virtual_state, self.optimizer._newoperations
@@ -147,6 +153,7 @@
             call_pure_results, True)
         jump_op = operations[-1]
         cell_token = jump_op.getdescr()
+        assert isinstance(cell_token, JitCellToken)
         if not inline_short_preamble or len(cell_token.target_tokens) == 1:
             return self.jump_to_preamble(cell_token, jump_op, info)
         # force all the information that does not go to the short
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -1,7 +1,7 @@
 from rpython.jit.metainterp.walkvirtual import VirtualVisitor
 from rpython.jit.metainterp.history import ConstInt, ConstPtr, ConstFloat
 from rpython.jit.metainterp.optimizeopt.info import ArrayPtrInfo,\
-     ArrayStructInfo
+     ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
      MININT, MAXINT, IntBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
@@ -175,6 +175,7 @@
     def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         box = optimizer.get_box_replacement(box)
         info = optimizer.getptrinfo(box)
+        assert isinstance(info, AbstractStructPtrInfo)
         if info is None or not info.is_virtual():
             raise BadVirtualState()
         for i in range(len(self.fielddescrs)):
@@ -255,7 +256,7 @@
         for i in range(len(self.fieldstate)):
             fieldbox = info.getitem(self.arraydescr, i)
             if fieldbox is None:
-                xxx
+                raise Exception("do we ever get here?")
                 v = value.get_missing_null_value()
             s = self.fieldstate[i]
             if s.position > self.position:
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -127,7 +127,8 @@
         if self.lenbound and self.lenbound.lower >= 1:
             if self.mode is mode_string:
                 lenop = ResOperation(rop.STRLEN, [op])
-            elif self.mode is mode_unicode:
+            else:
+                assert self.mode is mode_unicode
                 lenop = ResOperation(rop.UNICODELEN, [op])
             short.append(lenop)
             self.lenbound.make_guards(lenop, short)

From noreply at buildbot.pypy.org  Thu Aug 27 18:42:35 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Aug 2015 18:42:35 +0200 (CEST)
Subject: [pypy-commit] cffi default: Move this potential failure earlier
Message-ID: <20150827164235.BA62B1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2258:06cb88161224
Date: 2015-08-27 18:25 +0200
http://bitbucket.org/cffi/cffi/changeset/06cb88161224/

Log:	Move this potential failure earlier

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,6 +12,9 @@
 # ____________________________________________________________
 
 import sys
+assert __version__ == "1.2.1", ("This test_c.py file is for testing a version"
+                                " of cffi that differs from the one that we"
+                                " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
     type_or_class = "type"
     mandatory_b_prefix = ''
@@ -3435,7 +3438,3 @@
                             "be 'foo *', but the types are different (check "
                             "that you are not e.g. mixing up different ffi "
                             "instances)")
-
-def test_version():
-    # this test is here mostly for PyPy
-    assert __version__ == "1.2.1"

From noreply at buildbot.pypy.org  Thu Aug 27 18:43:21 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Thu, 27 Aug 2015 18:43:21 +0200 (CEST)
Subject: [pypy-commit] pypy default: Apply cffi's 06cb88161224
Message-ID: <20150827164321.53A311C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79261:0b377f21f77f
Date: 2015-08-27 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/0b377f21f77f/

Log:	Apply cffi's 06cb88161224

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,6 +1,9 @@
 # ____________________________________________________________
 
 import sys
+assert __version__ == "1.2.1", ("This test_c.py file is for testing a version"
+                                " of cffi that differs from the one that we"
+                                " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
     type_or_class = "type"
     mandatory_b_prefix = ''
@@ -3424,7 +3427,3 @@
                             "be 'foo *', but the types are different (check "
                             "that you are not e.g. mixing up different ffi "
                             "instances)")
-
-def test_version():
-    # this test is here mostly for PyPy
-    assert __version__ == "1.2.1"

From noreply at buildbot.pypy.org  Thu Aug 27 19:17:31 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Thu, 27 Aug 2015 19:17:31 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: packset combination now maximizes
 the packset and splits it right after combination into chunks that fit into
 the vector regsiter,
 this eliminates the sorting (lets see how this works out)
Message-ID: <20150827171731.3BFE61C050C@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79262:1b4b6db46742
Date: 2015-08-27 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/1b4b6db46742/

Log:	packset combination now maximizes the packset and splits it right
	after combination into chunks that fit into the vector regsiter,
	this eliminates the sorting (lets see how this works out)

diff --git a/rpython/jit/metainterp/optimizeopt/dependency.py b/rpython/jit/metainterp/optimizeopt/dependency.py
--- a/rpython/jit/metainterp/optimizeopt/dependency.py
+++ b/rpython/jit/metainterp/optimizeopt/dependency.py
@@ -1073,9 +1073,9 @@
 
     def __repr__(self):
         if self.is_identity():
-            return 'IndexVar(%s+%s)' % (self.var, repr(self.next_nonconst))
+            return 'idx(%s)' % (self.var,)
 
-        return 'IndexVar((%s*(%s/%s)+%s))' % (self.var, self.coefficient_mul,
+        return 'idx(%s*(%s/%s)+%s)' % (self.var, self.coefficient_mul,
                                             self.coefficient_div, self.constant)
 
 class MemoryRef(object):
diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -892,15 +892,21 @@
     def leftmost(self):
         return self.operations[0].getoperation()
 
+    def pack_type(self):
+        ptype = self.input_type
+        if self.input_type is None:
+            # load does not have an input type, but only an output type
+            ptype = self.output_type
+        return ptype
+
+    def pack_byte_size(self):
+        return self.pack_type().getsize() * self.opcount()
+
     def is_full(self, vec_reg_size):
         """ If one input element times the opcount is equal
             to the vector register size, we are full!
         """
-        ptype = self.input_type
-        if self.input_type is None:
-            # load does not have an input type, but only an output type
-            ptype = self.output_type
-
+        ptype = self.pack_type()
         op = self.leftmost()
         if op.casts_box():
             cur_bytes = ptype.getsize() * self.opcount()
@@ -908,7 +914,7 @@
             assert cur_bytes <= max_bytes
             return cur_bytes == max_bytes
 
-        bytes = ptype.getsize() * len(self.operations)
+        bytes = self.pack_byte_size()
         assert bytes <= vec_reg_size
         if bytes == vec_reg_size:
             return True
@@ -933,6 +939,20 @@
             node.pack = self
             node.pack_position = i
 
+    def split(self, packlist, vec_reg_size):
+        pack = self
+        pack_type = self.pack_type()
+        max_count = vec_reg_size // pack_type.getsize()
+        assert max_count * pack_type.getsize() == vec_reg_size
+        while pack.pack_byte_size() > vec_reg_size:
+            newpack = pack.clone()
+            oplist = pack.operations[:max_count]
+            newpack.operations = pack.operations[max_count:]
+            pack.operations = oplist
+            pack.update_pack_of_nodes()
+            newpack.update_pack_of_nodes()
+            pack = newpack
+
     def rightmost_match_leftmost(self, other):
         """ Check if pack A can be combined with pack B """
         assert isinstance(other, Pack)
@@ -954,6 +974,12 @@
     def is_accumulating(self):
         return self.accum is not None
 
+    def clone(self):
+        cloned = Pack(self.operations, self.input_type, self.output_type)
+        cloned.accum = self.accum
+        return cloned
+
+
 class Pair(Pack):
     """ A special Pack object with only two statements. """
     def __init__(self, left, right, input_type, output_type):
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -319,16 +319,16 @@
             for node_b,memref_b in memory_refs:
                 if memref_a is memref_b:
                     continue
+                #print "???", memref_a.index_var, memref_b.index_var
                 # instead of compare every possible combination and
                 # exclue a_opidx == b_opidx only consider the ones
                 # that point forward:
-                if node_a.is_before(node_b):
-                    if memref_a.is_adjacent_to(memref_b):
-                        pair = self.packset.can_be_packed(node_a, node_b, None, False)
-                        if pair:
-                            if node_a.op.getopnum() == rop.GETARRAYITEM_RAW:
-                                print "found", memref_a.index_var, memref_b.index_var
-                            self.packset.add_pack(pair)
+                if memref_a.is_adjacent_after(memref_b):
+                    pair = self.packset.can_be_packed(node_a, node_b, None, False)
+                    if pair:
+                        if node_a.op.getopnum() == rop.GETARRAYITEM_RAW:
+                            print " => found", memref_a.index_var, memref_b.index_var
+                        self.packset.add_pack(pair)
 
     def extend_packset(self):
         """ Follow dependency chains to find more candidates to put into
@@ -389,13 +389,13 @@
         """
         if len(self.packset.packs) == 0:
             raise NotAVectorizeableLoop()
-        packsort(self.packset.packs).sort()
-        if not we_are_translated():
-            # ensure we are really sorted!
-            x = 0
-            for i,pack in enumerate(self.packset.packs):
-                assert x <= pack.left.getindex()
-                x = pack.left.getindex()
+        #packsort(self.packset.packs).sort()
+        #if not we_are_translated():
+        #    # ensure we are really sorted!
+        #    x = 0
+        #    for i,pack in enumerate(self.packset.packs):
+        #        assert x <= pack.left.getindex()
+        #        x = pack.left.getindex()
         i = 0
         j = 0
         end_ij = len(self.packset.packs)
@@ -413,31 +413,31 @@
                     pack2 = self.packset.packs[j]
                     # remove intermediate
                     left = pack1.operations[0]
-                    if left in orphan:
-                        # a pack was filled, thus the rhs was put
-                        # into the orphan map.
-                        if orphan[left] is False:
-                            # this pack might be redundant if pack1.right
-                            # is the at the left position in another pack
-                            assert pack1.opcount() == 2
-                            right = pack1.operations[1]
-                            orphan[right] = True
-                            pack1.clear()
-                            del self.packset.packs[i]
-                            end_ij -= 1
-                            continue
-                        else:
-                            # left is not an orphan, this pack proves that
-                            # there might be more packs
-                            del orphan[left]
+                    #if left in orphan:
+                    #    # a pack was filled, thus the rhs was put
+                    #    # into the orphan map.
+                    #    if orphan[left] is False:
+                    #        # this pack might be redundant if pack1.right
+                    #        # is the at the left position in another pack
+                    #        assert pack1.opcount() == 2
+                    #        right = pack1.operations[1]
+                    #        orphan[right] = True
+                    #        pack1.clear()
+                    #        del self.packset.packs[i]
+                    #        end_ij -= 1
+                    #        continue
+                    #    else:
+                    #        # left is not an orphan, this pack proves that
+                    #        # there might be more packs
+                    #        del orphan[left]
                     # check if the pack is already full
-                    if pack1.is_full(self.cpu.vector_register_size):
-                        right = pack1.operations[-1]
-                        # False indicates that the next pair might not
-                        # be needed, because left is already computed
-                        # in another set
-                        orphan[right] = False
-                        break
+                    #if pack1.is_full(self.cpu.vector_register_size):
+                    #    right = pack1.operations[-1]
+                    #    # False indicates that the next pair might not
+                    #    # be needed, because left is already computed
+                    #    # in another set
+                    #    orphan[right] = False
+                    #    break
                     if pack1.rightmost_match_leftmost(pack2):
                         end_ij = self.packset.combine(i,j)
                     else:
@@ -449,7 +449,10 @@
             if len_before == len(self.packset.packs):
                 break
         for pack in self.packset.packs:
-            pack.update_pack_of_nodes()
+            if pack.pack_byte_size() > self.cpu.vector_register_size:
+                pack.split(self.packset.packs, self.cpu.vector_register_size)
+            else:
+                pack.update_pack_of_nodes()
 
         if not we_are_translated():
             # some test cases check the accumulation variables

From noreply at buildbot.pypy.org  Thu Aug 27 19:25:57 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Thu, 27 Aug 2015 19:25:57 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: rpythonize
Message-ID: <20150827172557.0501C1C0557@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79263:07045431ecaa
Date: 2015-08-27 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/07045431ecaa/

Log:	rpythonize

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -462,7 +462,7 @@
                 optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
-    def setitem(self, descr, index, struct_op, op, cf=None, optheap=None):
+    def setitem(self, descr, index, struct, op, cf=None, optheap=None):
         if self._items is None:
             self._items = [None] * (index + 1)
         if index >= len(self._items):
@@ -470,7 +470,7 @@
         self._items[index] = op
         if cf is not None:
             assert not self.is_virtual()
-            cf.register_dirty_field(struct_op, self)
+            cf.register_dirty_field(struct, self)
 
     def getitem(self, descr, index, optheap=None):
         if self._items is None or index >= len(self._items):
diff --git a/rpython/jit/metainterp/optimizeopt/intutils.py b/rpython/jit/metainterp/optimizeopt/intutils.py
--- a/rpython/jit/metainterp/optimizeopt/intutils.py
+++ b/rpython/jit/metainterp/optimizeopt/intutils.py
@@ -227,6 +227,7 @@
         return True
 
     def contains_bound(self, other):
+        assert isinstance(other, IntBound)
         if other.has_lower:
             if not self.contains(other.lower):
                 return False
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -266,6 +266,7 @@
                 if op.is_guard():
                     op = self.replace_op_with(op, op.getopnum(),
                                     descr=compile.ResumeAtPositionDescr())
+                    assert isinstance(op, GuardResOp)
                     op.rd_snapshot = patchguardop.rd_snapshot
                     op.rd_frame_info_list = patchguardop.rd_frame_info_list
                 self.optimizer.send_extra_operation(op)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -105,7 +105,7 @@
                 'Generating guards for making the VirtualStates ' +
                 'at hand match have not been implemented')
 
-    def enum_forced_boxes(self, boxes, structbox, optimizer, force_boxes=False):
+    def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
         raise NotImplementedError
 
     def enum(self, virtual_state):
@@ -148,6 +148,7 @@
         opinfo = state.optimizer.getptrinfo(box)
         if runtime_box is not None:
             assert opinfo.is_virtual()
+        assert isinstance(opinfo, AbstractStructPtrInfo)
 
         if len(self.fielddescrs) != len(other.fielddescrs):
             raise VirtualStatesCantMatch("field descrs don't match")
@@ -293,6 +294,7 @@
         fieldbox = None
         fieldbox_runtime = None
         opinfo = state.optimizer.getptrinfo(box)
+        assert isinstance(opinfo, ArrayPtrInfo)
         for i in range(self.length):
             for descr in self.fielddescrs:
                 index = i * len(self.fielddescrs) + descr.get_index()
@@ -311,8 +313,8 @@
             if s is not None:
                 s.enum(virtual_state)
 
-    def enum_forced_boxes(self, boxes, structbox, optimizer, force_boxes=False):
-        opinfo = optimizer.getptrinfo(structbox)
+    def enum_forced_boxes(self, boxes, box, optimizer, force_boxes=False):
+        opinfo = optimizer.getptrinfo(box)
         if not isinstance(opinfo, ArrayStructInfo):
             raise BadVirtualState
         if not opinfo.is_virtual():
@@ -360,7 +362,6 @@
                     self.level = LEVEL_KNOWNCLASS
                 elif info.is_nonnull():
                     self.level = LEVEL_NONNULL
-                # XXX strings?
                 self.lenbound = info.getlenbound(None)
         elif type == 'i':
             if isinstance(info, IntBound):

From noreply at buildbot.pypy.org  Fri Aug 28 09:13:03 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 09:13:03 +0200 (CEST)
Subject: [pypy-commit] pypy ppc-updated-backend: PPC Backend #1: merged
 "default" and the unfinished
Message-ID: <20150828071303.0CA0E1C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: ppc-updated-backend
Changeset: r79264:2b7e225663d7
Date: 2015-08-22 05:41 -0700
http://bitbucket.org/pypy/pypy/changeset/2b7e225663d7/

Log:	PPC Backend #1: merged "default" and the unfinished "ppc-updated-
	backend" branch, and then fixed stuff until we reach a state where
	it doesn't completely segfaults whenever we try to run any test. At
	this point, a bit more than half the tests of test_runner.py pass.

diff --git a/rpython/jit/backend/ppc/TODO b/rpython/jit/backend/ppc/TODO
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/ppc/TODO
@@ -0,0 +1,4 @@
+
+prepare_guard_int_lt & friends: re-enable in walk_operations()
+
+guard_xyz: fail if the target of the branch is too far away (+32k?)
diff --git a/rpython/jit/backend/ppc/arch.py b/rpython/jit/backend/ppc/arch.py
--- a/rpython/jit/backend/ppc/arch.py
+++ b/rpython/jit/backend/ppc/arch.py
@@ -72,10 +72,3 @@
 
 STD_FRAME_SIZE_IN_BYTES = GPR_SAVE_AREA_OFFSET + len(REGISTERS_SAVED) * WORD
 assert STD_FRAME_SIZE_IN_BYTES % 16 == 0
-
-
-# The JITFRAME_FIXED_SIZE is measured in words, and should be the
-# number of registers that need to be saved into the jitframe when
-# failing a guard, for example.  (Note: it is about the jitframe,
-# not the frame.)
-JITFRAME_FIXED_SIZE = len(r.MANAGED_REGS) + len(r.MANAGED_FP_REGS)
diff --git a/rpython/jit/backend/ppc/codebuilder.py b/rpython/jit/backend/ppc/codebuilder.py
--- a/rpython/jit/backend/ppc/codebuilder.py
+++ b/rpython/jit/backend/ppc/codebuilder.py
@@ -923,15 +923,15 @@
     def flush_icache(x, y): pass
 
 class PPCGuardToken(GuardToken):
-    # We may have to find a suitable default value for fcond
-    def __init__(self, cpu, gcmap, descr, failargs, faillocs, offset,
+    def __init__(self, cpu, gcmap, descr, failargs, faillocs,
                  exc, frame_depth, is_guard_not_invalidated=False,
-                 is_guard_not_forced=False, fcond=c.EQ):
+                 is_guard_not_forced=False, fcond=c.UH):
+        assert fcond != c.UH
         GuardToken.__init__(self, cpu, gcmap, descr, failargs, faillocs, exc,
                             frame_depth, is_guard_not_invalidated,
                             is_guard_not_forced)
         self.fcond = fcond
-        self.offset = offset
+        #self.offset = offset
 
 class OverwritingBuilder(PPCAssembler):
     def __init__(self, cb, start, num_insts):
@@ -990,14 +990,6 @@
         else:
             self.ldx(rD.value, 0, rD.value)
 
-    def store_reg(self, source_reg, addr):
-        with scratch_reg(self):
-            self.load_imm(r.SCRATCH, addr)
-            if IS_PPC_32:
-                self.stwx(source_reg.value, 0, r.SCRATCH.value)
-            else:
-                self.stdx(source_reg.value, 0, r.SCRATCH.value)
-
     def b_offset(self, target):
         curpos = self.currpos()
         offset = target - curpos
@@ -1160,7 +1152,7 @@
         self.writechar(chr(word & 0xFF))
 
     def currpos(self):
-        return self.get_rel_pos()
+        return self.get_relative_pos()
 
     def flush_cache(self, addr):
         startaddr = rffi.cast(lltype.Signed, addr)
diff --git a/rpython/jit/backend/ppc/condition.py b/rpython/jit/backend/ppc/condition.py
--- a/rpython/jit/backend/ppc/condition.py
+++ b/rpython/jit/backend/ppc/condition.py
@@ -9,6 +9,7 @@
 LT = (0, SET)
 EQ = (2, SET)
 GE = (0, UNSET)
+UH = (-1, -1)    # invalid
 
 # values below are random ...
 
diff --git a/rpython/jit/backend/ppc/jump.py b/rpython/jit/backend/ppc/jump.py
--- a/rpython/jit/backend/ppc/jump.py
+++ b/rpython/jit/backend/ppc/jump.py
@@ -1,7 +1,5 @@
 # ../x86/jump.py
 # XXX combine with ../x86/jump.py and move to llsupport
-import sys
-from rpython.tool.pairtype import extendabletype
 
 def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg):
     pending_dests = len(dst_locations)
@@ -77,9 +75,9 @@
 def remap_frame_layout_mixed(assembler,
                              src_locations1, dst_locations1, tmpreg1,
                              src_locations2, dst_locations2, tmpreg2):
-    # find and push the xmm stack locations from src_locations2 that
+    # find and push the fp stack locations from src_locations2 that
     # are going to be overwritten by dst_locations1
-    from pypy.jit.backend.ppc.arch import WORD
+    from rpython.jit.backend.ppc.arch import WORD
     extrapushes = []
     dst_keys = {}
     for loc in dst_locations1:
@@ -104,10 +102,10 @@
     # remap the integer and pointer registers and stack locations
     remap_frame_layout(assembler, src_locations1, dst_locations1, tmpreg1)
     #
-    # remap the vfp registers and stack locations
+    # remap the fp registers and stack locations
     remap_frame_layout(assembler, src_locations2, dst_locations2, tmpreg2)
     #
-    # finally, pop the extra xmm stack locations
+    # finally, pop the extra fp stack locations
     while len(extrapushes) > 0:
         loc = extrapushes.pop()
         assembler.regalloc_pop(loc)
diff --git a/rpython/jit/backend/ppc/locations.py b/rpython/jit/backend/ppc/locations.py
--- a/rpython/jit/backend/ppc/locations.py
+++ b/rpython/jit/backend/ppc/locations.py
@@ -1,9 +1,7 @@
 from rpython.jit.metainterp.history import INT, FLOAT
 import sys
 
-# TODO: solve the circular import: runner -> arch -> register -> locations ->
-# arch
-# XXX import from arch.py, currently we have a circular import
+# cannot import from arch.py, currently we have a circular import
 if sys.maxint == (2**31 - 1):
     WORD = 4
     FWORD = 8
@@ -12,8 +10,6 @@
     FWORD = 8
 DWORD = 2 * WORD
 
-# JITFRAME_FIXED_SIZE is also duplicated because of the circular import
-JITFRAME_FIXED_SIZE = 27 + 31 + 1 + 4 + 1
 
 class AssemblerLocation(object):
     _immutable_ = True
@@ -148,11 +144,8 @@
 def imm(val):
     return ImmLocation(val)
 
-def get_spp_offset(pos):
-    if pos < 0:
-        return -pos * WORD
-    else:
-        return -(pos + 1) * WORD
-
 def get_fp_offset(base_ofs, position):
-    return base_ofs + position
+    from rpython.jit.backend.ppc.register import JITFRAME_FIXED_SIZE
+    # Argument is a frame position (0, 1, 2...).
+    # Returns the n'th word beyond the fixed frame size.
+    return base_ofs + WORD * (position + JITFRAME_FIXED_SIZE)
diff --git a/rpython/jit/backend/ppc/opassembler.py b/rpython/jit/backend/ppc/opassembler.py
--- a/rpython/jit/backend/ppc/opassembler.py
+++ b/rpython/jit/backend/ppc/opassembler.py
@@ -211,24 +211,24 @@
     def emit_cast_float_to_int(self, op, arglocs, regalloc):
         l0, temp_loc, res = arglocs
         self.mc.fctidz(temp_loc.value, l0.value)
-        self.mc.stfd(temp_loc.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
-        self.mc.ld(res.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
+        self.mc.stfd(temp_loc.value, r.SP.value, -16)
+        self.mc.ld(res.value, r.SP.value, -16)
 
     def emit_cast_int_to_float(self, op, arglocs, regalloc):
         l0, temp_loc, res = arglocs
-        self.mc.std(l0.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
-        self.mc.lfd(temp_loc.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
+        self.mc.std(l0.value, r.SP.value, -16)
+        self.mc.lfd(temp_loc.value, r.SP.value, -16)
         self.mc.fcfid(res.value, temp_loc.value)
 
     def emit_convert_float_bytes_to_longlong(self, op, arglocs, regalloc):
         l0, res = arglocs
-        self.mc.stfd(l0.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
-        self.mc.ld(res.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
+        self.mc.stfd(l0.value, r.SP.value, -16)
+        self.mc.ld(res.value, r.SP.value, -16)
 
     def emit_convert_longlong_bytes_to_float(self, op, arglocs, regalloc):
         l0, res = arglocs
-        self.mc.std(l0.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
-        self.mc.lfd(res.value, r.SPP.value, FORCE_INDEX_OFS + WORD)
+        self.mc.std(l0.value, r.SP.value, -16)
+        self.mc.lfd(res.value, r.SP.value, -16)
 
 class GuardOpAssembler(object):
 
@@ -237,22 +237,22 @@
     def _emit_guard(self, op, arglocs, fcond, save_exc=False,
                     is_guard_not_invalidated=False,
                     is_guard_not_forced=False):
-        pos = self.mc.currpos()
-        self.mc.nop()     # has to be patched later on
         token = self.build_guard_token(op, arglocs[0].value, arglocs[1:],
                                        fcond, save_exc, is_guard_not_invalidated,
                                        is_guard_not_forced)
-        self.pending_guards.append(token)
+        token.pos_jump_offset = self.mc.currpos()
+        self.mc.nop()     # has to be patched later on
+        self.pending_guard_tokens.append(token)
 
     def build_guard_token(self, op, frame_depth, arglocs, fcond, save_exc,
                           is_guard_not_invalidated=False,
                           is_guard_not_forced=False):
         descr = op.getdescr()
-        offset = self.mc.currpos()
-        gcmap = allocate_gcmap(self, frame_depth, JITFRAME_FIXED_SIZE)
+        gcmap = allocate_gcmap(self, frame_depth, r.JITFRAME_FIXED_SIZE)
         token = PPCGuardToken(self.cpu, gcmap, descr, op.getfailargs(),
                               arglocs, save_exc, frame_depth,
-                              is_guard_not_invalidated, is_guard_not_forced)
+                              is_guard_not_invalidated, is_guard_not_forced,
+                              fcond)
         return token
 
     def emit_guard_true(self, op, arglocs, regalloc):
@@ -356,7 +356,10 @@
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         if len(arglocs) == 2:
             [return_val, fail_descr_loc] = arglocs
-            self.mc.std(return_val.value, r.SPP.value, base_ofs)
+            if op.getarg(0).type == FLOAT:
+                self.mc.stfd(return_val.value, r.SPP.value, base_ofs)
+            else:
+                self.mc.std(return_val.value, r.SPP.value, base_ofs)
         else:
             [fail_descr_loc] = arglocs
 
diff --git a/rpython/jit/backend/ppc/ppc_assembler.py b/rpython/jit/backend/ppc/ppc_assembler.py
--- a/rpython/jit/backend/ppc/ppc_assembler.py
+++ b/rpython/jit/backend/ppc/ppc_assembler.py
@@ -7,12 +7,12 @@
                                           LR_BC_OFFSET, REGISTERS_SAVED,
                                           GPR_SAVE_AREA_OFFSET,
                                           THREADLOCAL_ADDR_OFFSET,
-                                          STD_FRAME_SIZE_IN_BYTES,
-                                          JITFRAME_FIXED_SIZE)
+                                          STD_FRAME_SIZE_IN_BYTES)
 from rpython.jit.backend.ppc.helper.assembler import Saved_Volatiles
 from rpython.jit.backend.ppc.helper.regalloc import _check_imm_arg
 import rpython.jit.backend.ppc.register as r
 import rpython.jit.backend.ppc.condition as c
+from rpython.jit.backend.ppc.register import JITFRAME_FIXED_SIZE
 from rpython.jit.metainterp.history import AbstractFailDescr
 from rpython.jit.metainterp.history import ConstInt, BoxInt
 from rpython.jit.backend.llsupport import jitframe
@@ -30,7 +30,7 @@
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib.objectmodel import we_are_translated, specialize
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.jit.backend.ppc.locations import StackLocation, get_spp_offset, imm
+from rpython.jit.backend.ppc.locations import StackLocation, get_fp_offset, imm
 from rpython.rlib.jit import AsmInfo
 from rpython.rlib.objectmodel import compute_unique_id
 from rpython.rlib.rarithmetic import r_uint
@@ -166,50 +166,29 @@
         mc.addi(r.r15.value, r.r15.value, -2 * WORD)  # SUB r15, r15, 2*WORD
         mc.store(r.r15.value, r.r14.value, 0) # STR r15, [rootstacktop]
 
+    def new_stack_loc(self, i, tp):
+        base_ofs = self.cpu.get_baseofs_of_frame_field()
+        return StackLocation(i, get_fp_offset(base_ofs, i), tp)
+
     def setup_failure_recovery(self):
-
-        @rgc.no_collect
-        def failure_recovery_func(mem_loc, spilling_pointer,
-                                  managed_registers_pointer):
-            """
-                mem_loc is a pointer to the beginning of the encoding.
-
-                spilling_pointer is the address of the spilling area.
-            """
-            regs = rffi.cast(rffi.LONGP, managed_registers_pointer)
-            fpregs = rffi.ptradd(regs, len(r.MANAGED_REGS))
-            fpregs = rffi.cast(rffi.LONGP, fpregs)
-            return self.decode_registers_and_descr(mem_loc, 
-                                                   spilling_pointer,
-                                                   regs, fpregs)
-
-        self.failure_recovery_func = failure_recovery_func
-        self.failure_recovery_code = [0, 0, 0]
-
-    recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3,
-            lltype.Signed))
+        self.failure_recovery_code = [0, 0, 0, 0]
 
     # TODO: see with we really need the ignored_regs argument
     def _push_all_regs_to_jitframe(self, mc, ignored_regs, withfloats,
                                    callee_only=False):
         base_ofs = self.cpu.get_baseofs_of_frame_field()
         if callee_only:
-            # Only push registers used to pass arguments to the callee
-            regs = r.VOLATILES
+            regs = XXX
         else:
-            regs = r.ALL_REGS
+            regs = r.MANAGED_REGS
         # For now, just push all regs to the jitframe
-        for i, reg in enumerate(regs):
-            # XXX should we progress to higher addresses?
-            mc.store_reg(reg, base_ofs - (i * WORD))
-
+        for reg in regs:
+            v = r.ALL_REG_INDEXES[reg]
+            mc.std(reg.value, r.SPP.value, base_ofs + v * WORD)
         if withfloats:
-            if callee_only:
-                regs = r.VOLATILES_FLOAT
-            else:
-                regs = r.ALL_FLOAT_REGS
-            for i, reg in enumerate(regs):
-                pass # TODO find or create the proper store indexed for fpr's
+            for reg in r.MANAGED_FP_REGS:
+                v = r.ALL_REG_INDEXES[reg]
+                mc.stfd(reg.value, r.SPP.value, base_ofs + v * WORD)
 
     def _pop_all_regs_from_jitframe(self, mc, ignored_regs, withfloats,
                                     callee_only=False):
@@ -230,135 +209,36 @@
             for i, reg in enumerate(regs):
                 pass # TODO find or create the proper load indexed for fpr's
 
-    @rgc.no_collect
-    def decode_registers_and_descr(self, mem_loc, spp, registers, fp_registers):
-        """Decode locations encoded in memory at mem_loc and write the values
-        to the failboxes.  Values for spilled vars and registers are stored on
-        stack at frame_loc """
-        assert spp & 1 == 0
-        self.fail_force_index = spp + FORCE_INDEX_OFS
-        bytecode = rffi.cast(rffi.UCHARP, mem_loc)
-        num = 0
-        value = 0
-        fvalue = 0
-        code_inputarg = False
-        while True:
-            code = rffi.cast(lltype.Signed, bytecode[0])
-            bytecode = rffi.ptradd(bytecode, 1)
-            if code >= self.CODE_FROMSTACK:
-                if code > 0x7F:
-                    shift = 7
-                    code &= 0x7F
-                    while True:
-                        nextcode = rffi.cast(lltype.Signed, bytecode[0])
-                        bytecode = rffi.ptradd(bytecode, 1)
-                        code |= (nextcode & 0x7F) << shift
-                        shift += 7
-                        if nextcode <= 0x7F:
-                            break
-                # load the value from the stack
-                kind = code & 3
-                code = int((code - self.CODE_FROMSTACK) >> 2)
-                if code_inputarg:
-                    code = ~code
-                    code_inputarg = False
-                if kind == self.DESCR_FLOAT:
-                    start = spp + get_spp_offset(int(code))
-                    fvalue = rffi.cast(rffi.LONGP, start)[0]
-                else:
-                    start = spp + get_spp_offset(int(code))
-                    value = rffi.cast(rffi.LONGP, start)[0]
-            else:
-                # 'code' identifies a register: load its value
-                kind = code & 3
-                if kind == self.DESCR_SPECIAL:
-                    if code == self.CODE_HOLE:
-                        num += 1
-                        continue
-                    if code == self.CODE_INPUTARG:
-                        code_inputarg = True
-                        continue
-                    assert code == self.CODE_STOP
-                    break
-                code >>= 2
-                if kind == self.DESCR_FLOAT:
-                    reg_index = r.get_managed_fpreg_index(code)
-                    fvalue = fp_registers[reg_index]
-                else:
-                    reg_index = r.get_managed_reg_index(code)
-                    value = registers[reg_index]
-            # store the loaded value into fail_boxes_
-            if kind == self.DESCR_FLOAT:
-                tgt = self.fail_boxes_float.get_addr_for_num(num)
-                rffi.cast(rffi.LONGP, tgt)[0] = fvalue
-            else:
-                if kind == self.DESCR_INT:
-                    tgt = self.fail_boxes_int.get_addr_for_num(num)
-                elif kind == self.DESCR_REF:
-                    assert (value & 3) == 0, "misaligned pointer"
-                    tgt = self.fail_boxes_ptr.get_addr_for_num(num)
-                else:
-                    assert 0, "bogus kind"
-                rffi.cast(rffi.LONGP, tgt)[0] = value
-            num += 1
-        self.fail_boxes_count = num
-        fail_index = rffi.cast(rffi.INTP, bytecode)[0]
-        fail_index = rffi.cast(lltype.Signed, fail_index)
-        return fail_index
+    def _build_failure_recovery(self, exc, withfloats=False):
+        mc = PPCBuilder()
+        self.mc = mc
 
-    def decode_inputargs(self, code):
-        descr_to_box_type = [REF, INT, FLOAT]
-        bytecode = rffi.cast(rffi.UCHARP, code)
-        arglocs = []
-        code_inputarg = False
-        while 1:
-            # decode the next instruction from the bytecode
-            code = rffi.cast(lltype.Signed, bytecode[0])
-            bytecode = rffi.ptradd(bytecode, 1)
-            if code >= self.CODE_FROMSTACK:
-                # 'code' identifies a stack location
-                if code > 0x7F:
-                    shift = 7
-                    code &= 0x7F
-                    while True:
-                        nextcode = rffi.cast(lltype.Signed, bytecode[0])
-                        bytecode = rffi.ptradd(bytecode, 1)
-                        code |= (nextcode & 0x7F) << shift
-                        shift += 7
-                        if nextcode <= 0x7F:
-                            break
-                kind = code & 3
-                code = (code - self.CODE_FROMSTACK) >> 2
-                if code_inputarg:
-                    code = ~code
-                    code_inputarg = False
-                loc = PPCFrameManager.frame_pos(code, descr_to_box_type[kind])
-            elif code == self.CODE_STOP:
-                break
-            elif code == self.CODE_HOLE:
-                continue
-            elif code == self.CODE_INPUTARG:
-                code_inputarg = True
-                continue
-            else:
-                # 'code' identifies a register
-                kind = code & 3
-                code >>= 2
-                if kind == self.DESCR_FLOAT:
-                    assert (r.ALL_FLOAT_REGS[code] is 
-                            r.MANAGED_FP_REGS[r.get_managed_fpreg_index(code)])
-                    loc = r.ALL_FLOAT_REGS[code]
-                else:
-                    #loc = r.all_regs[code]
-                    assert (r.ALL_REGS[code] is 
-                            r.MANAGED_REGS[r.get_managed_reg_index(code)])
-                    loc = r.ALL_REGS[code]
-            arglocs.append(loc)
-        return arglocs[:]
+        # fill in the jf_descr and jf_gcmap fields of the frame according
+        # to which failure we are resuming from.  These are set before
+        # this function is called (see generate_quick_failure()).
+        ofs = self.cpu.get_ofs_of_frame_field('jf_descr')
+        ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap')
+        mc.store(r.r0.value, r.SPP.value, ofs)
+        mc.store(r.r2.value, r.SPP.value, ofs2)
 
-    # TODO
-    def _build_failure_recovery(self, exc, withfloats=False):
-        pass
+        self._push_all_regs_to_jitframe(mc, [], withfloats)
+
+        if exc:
+            # We might have an exception pending.  Load it into r2...
+            mc.write32(0)
+            #mc.MOV(ebx, heap(self.cpu.pos_exc_value()))
+            #mc.MOV(heap(self.cpu.pos_exception()), imm0)
+            #mc.MOV(heap(self.cpu.pos_exc_value()), imm0)
+            ## ...and save ebx into 'jf_guard_exc'
+            #offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc')
+            #mc.MOV_br(offset, ebx.value)
+
+        # now we return from the complete frame, which starts from
+        # _call_header_with_stack_check().  The _call_footer below does it.
+        self._call_footer()
+        rawstart = mc.materialize(self.cpu, [])
+        self.failure_recovery_code[exc + 2 * withfloats] = rawstart
+        self.mc = None
 
     # TODO
     def build_frame_realloc_slowpath(self):
@@ -836,7 +716,7 @@
         #
         self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
                                 rawstart)
-        looptoken._ll_loop_code = looppos + rawstart
+        looptoken._ppc_loop_code = looppos + rawstart
         debug_start("jit-backend-addr")
         debug_print("Loop %d (%s) has address 0x%x to 0x%x (bootstrap 0x%x)" % (
             looptoken.number, loopname,
@@ -881,56 +761,48 @@
             frame_depth = max(frame_depth, target_frame_depth)
         return frame_depth
 
-    def assemble_bridge(self, faildescr, inputargs, operations, looptoken, log):
+    @rgc.no_release_gil
+    def assemble_bridge(self, faildescr, inputargs, operations,
+                        original_loop_token, log, logger):
         if not we_are_translated():
+            # Arguments should be unique
             assert len(set(inputargs)) == len(inputargs)
 
-        self.setup(looptoken)
+        self.setup(original_loop_token)
         descr_number = compute_unique_id(faildescr)
         if log:
             operations = self._inject_debugging_code(faildescr, operations,
                                                      'b', descr_number)
-        assert isinstance(faildescr, AbstractFailDescr)
+
         arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs)
-
         regalloc = Regalloc(assembler=self)
+        startpos = self.mc.get_relative_pos()
         operations = regalloc.prepare_bridge(inputargs, arglocs,
                                              operations,
                                              self.current_clt.allgcrefs,
                                              self.current_clt.frame_info)
-
-        startpos = self.mc.currpos()
-        spilling_area, param_depth = self._assemble(operations, regalloc)
-        codeendpos = self.mc.currpos()
-
+        #self._check_frame_depth(self.mc, regalloc.get_gcmap())
+        frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations)
+        codeendpos = self.mc.get_relative_pos()
         self.write_pending_failure_recoveries()
-
-        rawstart = self.materialize_loop(looptoken, False)
-        self.process_pending_guards(rawstart)
-        self.patch_trace(faildescr, looptoken, rawstart, regalloc)
+        fullsize = self.mc.get_relative_pos()
+        #
+        rawstart = self.materialize_loop(original_loop_token)
+        self.patch_stack_checks(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE,
+                                rawstart)
+        debug_bridge(descr_number, rawstart, codeendpos)
+        self.patch_pending_failure_recoveries(rawstart)
+        # patch the jump from original guard
+        self.patch_jump_for_descr(faildescr, rawstart)
+        ops_offset = self.mc.ops_offset
+        frame_depth = max(self.current_clt.frame_info.jfi_frame_depth,
+                          frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE)
+        if logger:
+            logger.log_bridge(inputargs, operations, "rewritten",
+                              ops_offset=ops_offset)
         self.fixup_target_tokens(rawstart)
-        self.current_clt.frame_depth = max(self.current_clt.frame_depth,
-                spilling_area)
-        self.current_clt.param_depth = max(self.current_clt.param_depth, param_depth)
-
-        if not we_are_translated():
-            # for the benefit of tests
-            faildescr._ppc_bridge_frame_depth = self.current_clt.frame_depth
-            faildescr._ppc_bridge_param_depth = self.current_clt.param_depth
-            if log:
-                self.mc._dump_trace(rawstart, 'bridge_%d.asm' %
-                self.cpu.total_compiled_bridges)
-
-        self._patch_sp_offset(sp_patch_location, rawstart)
-
-        ops_offset = self.mc.ops_offset
+        self.update_frame_depth(frame_depth)
         self.teardown()
-
-        debug_start("jit-backend-addr")
-        debug_print("bridge out of Guard %d has address %x to %x" %
-                    (descr_number, rawstart, rawstart + codeendpos))
-        debug_stop("jit-backend-addr")
-
         return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos)
 
     def _patch_sp_offset(self, sp_patch_location, rawstart):
@@ -1053,21 +925,20 @@
         #print "=== Loop start is at %s ===" % hex(r_uint(start))
         return start
 
-    def push_gcmap(self, mc, gcmap, push=False, store=False):
+    def load_gcmap(self, mc, gcmap):
+        # load the current gcmap into register r2
         ptr = rffi.cast(lltype.Signed, gcmap)
-        if push:
-            with scratch_reg(mc):
-                mc.load_imm(r.SCRATCH, ptr)
-                mc.stdu(r.SCRATCH.value, r.SP.value, -WORD)
-        elif store:
-            assert False, "Not implemented"
+        mc.load_imm(r.r2, ptr)
 
     def generate_quick_failure(self, guardtok):
         startpos = self.mc.currpos()
         fail_descr, target = self.store_info_on_descr(startpos, guardtok)
-        self.regalloc_push(imm(fail_descr))
-        self.push_gcmap(self.mc, gcmap=guardtok.gcmap, push=True)
-        self.mc.call(target)
+        assert target != 0
+        self.load_gcmap(self.mc, gcmap=guardtok.gcmap)   # -> r2
+        self.mc.load_imm(r.r0, target)
+        self.mc.mtctr(r.r0.value)
+        self.mc.load_imm(r.r0, fail_descr)
+        self.mc.bctr()
         return startpos
 
     def write_pending_failure_recoveries(self):
@@ -1079,32 +950,37 @@
     def patch_pending_failure_recoveries(self, rawstart):
         clt = self.current_clt
         for tok in self.pending_guard_tokens:
-            xxxxxxxxx
-
-    def process_pending_guards(self, block_start):
-        clt = self.current_clt
-        for tok in self.pending_guards:
-            descr = tok.faildescr
-            assert isinstance(descr, AbstractFailDescr)
-            descr._ppc_block_start = block_start
-
+            addr = rawstart + tok.pos_jump_offset
+            #
+            # XXX see patch_jump_for_descr()
+            #tok.faildescr.adr_jump_offset = addr
+            tok.faildescr.adr_recovery_stub = rawstart + tok.pos_recovery_stub
+            #
+            relative_target = tok.pos_recovery_stub - tok.pos_jump_offset
+            #
             if not tok.is_guard_not_invalidated:
                 mc = PPCBuilder()
-                offset = tok.pos_recovery_stub - tok.offset
-                mc.b_cond_offset(offset, tok.fcond)
-                mc.copy_to_raw_memory(block_start + tok.offset)
+                mc.b_cond_offset(relative_target, tok.fcond)
+                mc.copy_to_raw_memory(addr)
             else:
-                clt.invalidate_positions.append((block_start + tok.offset,
-                        descr._ppc_guard_pos - tok.offset))
+                # GUARD_NOT_INVALIDATED, record an entry in
+                # clt.invalidate_positions of the form:
+                #     (addr-in-the-code-of-the-not-yet-written-jump-target,
+                #      relative-target-to-use)
+                relpos = tok.pos_jump_offset
+                clt.invalidate_positions.append((rawstart + relpos,
+                                                 relative_target))
 
-    def patch_trace(self, faildescr, looptoken, bridge_addr, regalloc):
-        # The first instruction (word) is not overwritten, because it is the
-        # one that actually checks the condition
+    def patch_jump_for_descr(self, faildescr, adr_new_target):
+        # 'faildescr.adr_jump_offset' is the address of an instruction that is a
+        # conditional jump.  We must patch this conditional jump to go
+        # to 'adr_new_target'.  If the target is too far away, we can't
+        # patch it inplace, and instead we patch the quick failure code
+        # (which should be at least 5 instructions, so enough).
+        # --- XXX for now we always use the second solution ---
         mc = PPCBuilder()
-        patch_addr = faildescr._ppc_block_start + faildescr._ppc_guard_pos
-        mc.b_abs(bridge_addr)
-        mc.copy_to_raw_memory(patch_addr)
-        faildescr._failure_recovery_code_ofs = 0
+        mc.b_abs(adr_new_target)
+        mc.copy_to_raw_memory(faildescr.adr_recovery_stub)
 
     def get_asmmemmgr_blocks(self, looptoken):
         clt = looptoken.compiled_loop_token
diff --git a/rpython/jit/backend/ppc/regalloc.py b/rpython/jit/backend/ppc/regalloc.py
--- a/rpython/jit/backend/ppc/regalloc.py
+++ b/rpython/jit/backend/ppc/regalloc.py
@@ -5,7 +5,7 @@
 from rpython.jit.codewriter import longlong
 from rpython.jit.backend.ppc.jump import (remap_frame_layout,
                                           remap_frame_layout_mixed)
-from rpython.jit.backend.ppc.locations import imm, get_fp_offset, get_spp_offset
+from rpython.jit.backend.ppc.locations import imm, get_fp_offset
 from rpython.jit.backend.ppc.helper.regalloc import (_check_imm_arg,
                                                      prepare_cmp_op,
                                                      prepare_unary_int_op,
@@ -192,7 +192,7 @@
 
     @staticmethod
     def get_loc_index(loc):
-        assert loc.is_stack()
+        assert isinstance(loc, locations.StackLocation)
         return loc.position
 
 class Regalloc(BaseRegalloc):
@@ -227,28 +227,41 @@
         # note: we need to make a copy of inputargs because possibly_free_vars
         # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
+        self.min_bytes_before_label = 4    # for redirect_call_assembler()
         return operations
 
-    def prepare_bridge(self, inputargs, arglocs, ops):
-        self._prepare(inputargs, ops)
+    def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs,
+                       frame_info):
+        operations = self._prepare(inputargs, operations, allgcrefs)
         self._update_bindings(arglocs, inputargs)
+        self.min_bytes_before_label = 0
+        return operations
+
+    def ensure_next_label_is_at_least_at_position(self, at_least_position):
+        self.min_bytes_before_label = max(self.min_bytes_before_label,
+                                          at_least_position)
 
     def _update_bindings(self, locs, inputargs):
+        # XXX this should probably go to llsupport/regalloc.py
         used = {}
         i = 0
         for loc in locs:
+            if loc is None: # xxx bit kludgy
+                loc = r.SPP
             arg = inputargs[i]
             i += 1
             if loc.is_reg():
-                self.rm.reg_bindings[arg] = loc
+                if loc is r.SPP:
+                    self.rm.bindings_to_frame_reg[arg] = None
+                else:
+                    self.rm.reg_bindings[arg] = loc
+                    used[loc] = None
             elif loc.is_fp_reg():
                 self.fprm.reg_bindings[arg] = loc
+                used[loc] = None
             else:
                 assert loc.is_stack()
-                self.frame_manager.set_binding(arg, loc)
-            used[loc] = None
-
-        # XXX combine with x86 code and move to llsupport
+                self.fm.bind(arg, loc)
         self.rm.free_regs = []
         for reg in self.rm.all_regs:
             if reg not in used:
@@ -257,9 +270,10 @@
         for reg in self.fprm.all_regs:
             if reg not in used:
                 self.fprm.free_regs.append(reg)
-        # note: we need to make a copy of inputargs because possibly_free_vars
-        # is also used on op args, which is a non-resizable list
         self.possibly_free_vars(list(inputargs))
+        self.fm.finish_binding()
+        self.rm._check_invariants()
+        self.fprm._check_invariants()
 
     def get_final_frame_depth(self):
         return self.fm.get_frame_depth()
@@ -317,7 +331,12 @@
                 i += 1
                 self.possibly_free_vars_for_op(op)
                 continue
-            if self.can_merge_with_next_guard(op, i, operations):
+            if self.can_merge_with_next_guard(op, i, operations) and (
+                # XXX FIX
+                op.getopnum() in (rop.CALL_RELEASE_GIL, rop.CALL_ASSEMBLER,
+                                  rop.CALL_MAY_FORCE)
+                # XXX FIX
+                ):
                 arglocs = oplist_with_guard[op.getopnum()](self, op,
                                                            operations[i + 1])
                 assert arglocs is not None
@@ -339,11 +358,18 @@
             i += 1
         assert not self.rm.reg_bindings
         assert not self.fprm.reg_bindings
-        #self.flush_loop()
+        self.flush_loop()
         self.assembler.mc.mark_op(None) # end of the loop
         for arg in inputargs:
             self.possibly_free_var(arg)
 
+    def flush_loop(self):
+        # Emit a nop in the rare case where we have a guard_not_invalidated
+        # immediately before a label
+        mc = self.assembler.mc
+        while self.min_bytes_before_label > mc.get_relative_pos():
+            mc.nop()
+
     def loc(self, var):
         if var.type == FLOAT:
             return self.fprm.loc(var)
@@ -360,6 +386,10 @@
         else:
             self.rm.force_spill_var(var)
 
+    def _consider_force_spill(self, op):
+        # This operation is used only for testing
+        self.force_spill_var(op.getarg(0))
+
     def before_call(self, force_store=[], save_all_regs=False):
         self.rm.before_call(force_store, save_all_regs)
         self.fprm.before_call(force_store, save_all_regs)
@@ -561,12 +591,13 @@
     def _prepare_guard(self, op, args=None):
         if args is None:
             args = []
-        args.append(imm(len(self.frame_manager.used)))
+        args.append(imm(self.fm.get_frame_depth()))
         for arg in op.getfailargs():
             if arg:
                 args.append(self.loc(arg))
             else:
                 args.append(None)
+        self.possibly_free_vars(op.getfailargs())
         return args
     
     def prepare_guard_true(self, op):
@@ -695,7 +726,7 @@
             if isinstance(box, Box):
                 loc = arglocs[i]
                 if loc is not None and loc.is_stack():
-                    self.frame_manager.hint_frame_locations[box] = loc
+                    self.fm.hint_frame_pos[box] = self.fm.get_loc_index(loc)
 
     def prepare_jump(self, op):
         descr = op.getdescr()
@@ -1067,7 +1098,6 @@
         return [res_loc]
 
     def prepare_label(self, op):
-        # XXX big refactoring needed?
         descr = op.getdescr()
         assert isinstance(descr, TargetToken)
         inputargs = op.getarglist()
@@ -1082,15 +1112,26 @@
             assert isinstance(arg, Box)
             if self.last_real_usage.get(arg, -1) <= position:
                 self.force_spill_var(arg)
-
+        #
+        # we need to make sure that no variable is stored in spp (=r31)
+        for arg in inputargs:
+            if self.loc(arg) is r.SPP:
+                loc2 = self.fm.loc(arg)
+                self.assembler.mc.store(r.SPP, loc2)
+        self.rm.bindings_to_frame_reg.clear()
         #
         for i in range(len(inputargs)):
             arg = inputargs[i]
             assert isinstance(arg, Box)
             loc = self.loc(arg)
+            assert loc is not r.SPP
             arglocs[i] = loc
             if loc.is_reg():
-                self.frame_manager.mark_as_free(arg)
+                self.fm.mark_as_free(arg)
+        #
+        # if we are too close to the start of the loop, the label's target may
+        # get overridden by redirect_call_assembler().  (rare case)
+        self.flush_loop()
         #
         descr._ppc_arglocs = arglocs
         descr._ppc_loop_code = self.assembler.mc.currpos()
diff --git a/rpython/jit/backend/ppc/register.py b/rpython/jit/backend/ppc/register.py
--- a/rpython/jit/backend/ppc/register.py
+++ b/rpython/jit/backend/ppc/register.py
@@ -37,6 +37,16 @@
 
 MANAGED_FP_REGS = VOLATILES_FLOAT[1:] #+ NONVOLATILES_FLOAT
 
+
+# The JITFRAME_FIXED_SIZE is measured in words, and should be the
+# number of registers that need to be saved into the jitframe when
+# failing a guard, for example.
+ALL_REG_INDEXES = {}
+for _r in MANAGED_REGS + MANAGED_FP_REGS:
+    ALL_REG_INDEXES[_r] = len(ALL_REG_INDEXES)
+JITFRAME_FIXED_SIZE = len(ALL_REG_INDEXES)
+
+
 PARAM_REGS = [r3, r4, r5, r6, r7, r8, r9, r10]
 PARAM_FPREGS = [f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11, f12, f13]
 
diff --git a/rpython/jit/backend/ppc/runner.py b/rpython/jit/backend/ppc/runner.py
--- a/rpython/jit/backend/ppc/runner.py
+++ b/rpython/jit/backend/ppc/runner.py
@@ -18,8 +18,13 @@
     IS_64_BIT = True
     BOOTSTRAP_TP = lltype.FuncType([], lltype.Signed)
 
+    from rpython.jit.backend.ppc.register import JITFRAME_FIXED_SIZE
     frame_reg = r.SP
-    all_reg_indexes = range(len(r.ALL_REGS))
+    all_reg_indexes = [-1] * 32
+    for _i, _r in enumerate(r.MANAGED_REGS):
+        all_reg_indexes[_r.value] = _i
+    gen_regs = r.MANAGED_REGS
+    float_regs = r.MANAGED_FP_REGS
 
     def __init__(self, rtyper, stats, opts=None, translate_support_code=False,
                  gcdescr=None):
@@ -31,7 +36,7 @@
         AbstractLLCPU.__init__(self, rtyper, stats, opts,
                                translate_support_code, gcdescr)
 
-        # floats are not supported yet
+        # floats are supported.  singlefloats are not supported yet
         self.supports_floats = True
 
     def setup(self):
@@ -44,11 +49,11 @@
         self.assembler.finish_once()
 
     def compile_bridge(self, faildescr, inputargs, operations,
-                      original_loop_token, log=False):
+                       original_loop_token, log=True, logger=None):
         clt = original_loop_token.compiled_loop_token
         clt.compiling_a_bridge()
         return self.assembler.assemble_bridge(faildescr, inputargs, operations,
-                                       original_loop_token, log=log)
+                                              original_loop_token, log, logger)
 
     @staticmethod
     def cast_ptr_to_int(x):
diff --git a/rpython/jit/backend/ppc/test/test_runner.py b/rpython/jit/backend/ppc/test/test_runner.py
--- a/rpython/jit/backend/ppc/test/test_runner.py
+++ b/rpython/jit/backend/ppc/test/test_runner.py
@@ -76,10 +76,11 @@
         ARGS = [lltype.Signed] * numargs
         RES = lltype.Signed
         args = [i+1 for i in range(numargs)]
-        res = self.cpu.execute_token(looptoken, *args)
-        assert res is faildescr
+        deadframe = self.cpu.execute_token(looptoken, *args)
+        fail = self.cpu.get_latest_descr(deadframe)
+        assert fail is faildescr
         for i in range(numargs):
-            assert self.cpu.get_latest_value_int(i) == i + 1
+            assert self.cpu.get_int_value(deadframe, i) == i + 1
 
         bridgeops = [arglist]
         bridgeops.append("guard_value(i1, -5) %s" % arglist)
@@ -88,12 +89,14 @@
         faildescr2 = bridge.operations[-1].getdescr()
 
         self.cpu.compile_bridge(faildescr, bridge.inputargs, bridge.operations, looptoken)
-        res2 = self.cpu.execute_token(looptoken, *args)
-        assert res2 is faildescr2
+        deadframe = self.cpu.execute_token(looptoken, *args)
+        fail = self.cpu.get_latest_descr(deadframe)
+        assert fail is faildescr2
         for i in range(numargs):
-            assert self.cpu.get_latest_value_int(i) == i + 1
+            assert self.cpu.get_int_value(deadframe, i) == i + 1
 
     def test_unicodesetitem_really_needs_temploc(self):
+        py.test.skip("XXX")
         u_box = self.alloc_unicode(u"abcdsdasdsaddefg")
         
         i0 = BoxInt()
@@ -128,6 +131,7 @@
             assert self.cpu.get_latest_value_int(i) == args[i]
 
     def test_debugger_on(self):
+        py.test.skip("XXX")
         from pypy.rlib import debug
 
         targettoken, preambletoken = TargetToken(), TargetToken()
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -230,6 +230,8 @@
         self.cpu.compile_loop(inputargs, operations, looptoken)
         if hasattr(looptoken, '_x86_ops_offset'):
             del looptoken._x86_ops_offset # else it's kept alive
+        if hasattr(looptoken, '_ppc_ops_offset'):
+            del looptoken._ppc_ops_offset # else it's kept alive
         del i0, i1, i2
         del inputargs
         del operations

From noreply at buildbot.pypy.org  Fri Aug 28 10:42:43 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 10:42:43 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: boolify the immutable-ranking
Message-ID: <20150828084243.4EBDD1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79265:61cd3d2b9379
Date: 2015-08-28 10:42 +0200
http://bitbucket.org/pypy/pypy/changeset/61cd3d2b9379/

Log:	boolify the immutable-ranking

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -190,7 +190,7 @@
         flag = get_type_flag(FIELDTYPE)
         name = '%s.%s' % (STRUCT._name, fieldname)
         index_in_parent = heaptracker.get_fielddescr_index_in(STRUCT, fieldname)
-        is_pure = STRUCT._immutable_field(fieldname)
+        is_pure = bool(STRUCT._immutable_field(fieldname))
         fielddescr = FieldDescr(name, offset, size, flag, index_in_parent,
                                 is_pure)
         cachedict = cache.setdefault(STRUCT, {})
@@ -309,7 +309,7 @@
         else:
             lendescr = get_field_arraylen_descr(gccache, ARRAY_OR_STRUCT)
         flag = get_type_flag(ARRAY_INSIDE.OF)
-        is_pure = ARRAY_INSIDE._immutable_field(None)
+        is_pure = bool(ARRAY_INSIDE._immutable_field(None))
         arraydescr = ArrayDescr(basesize, itemsize, lendescr, flag, is_pure)
         cache[ARRAY_OR_STRUCT] = arraydescr
         if isinstance(ARRAY_INSIDE.OF, lltype.Struct):

From noreply at buildbot.pypy.org  Fri Aug 28 10:46:57 2015
From: noreply at buildbot.pypy.org (maxbelanger)
Date: Fri, 28 Aug 2015 10:46:57 +0200 (CEST)
Subject: [pypy-commit] cffi add-float-dotdotdot: Add support for `typedef
	float... foo_t`.
Message-ID: <20150828084657.73FF81C03C0@cobra.cs.uni-duesseldorf.de>

Author: Max Belanger 
Branch: add-float-dotdotdot
Changeset: r2259:ec78097f627d
Date: 2015-08-27 17:47 -0700
http://bitbucket.org/cffi/cffi/changeset/ec78097f627d/

Log:	Add support for `typedef float... foo_t`.

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -214,6 +214,11 @@
      (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) :    \
      _CFFI__UNKNOWN_PRIM)
 
+#define _cffi_prim_float(size)                                          \
+    ((size) == 4 ? _CFFI_PRIM_FLOAT :                                   \
+     (size) == 8 ? _CFFI_PRIM_DOUBLE :                                  \
+     _CFFI__UNKNOWN_PRIM)
+
 #define _cffi_check_int(got, got_nonpos, expected)      \
     ((got_nonpos) == (expected <= 0) &&                 \
      (got) == (unsigned long long)expected)
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -648,10 +648,18 @@
         assert typenames[-1] == '__dotdotdot__'
         if len(typenames) == 1:
             return model.unknown_type(decl.name)
+
+        result = None
         for t in typenames[:-1]:
-            if t not in ['int', 'short', 'long', 'signed', 'unsigned', 'char']:
+            if t in ('int', 'short', 'long', 'signed', 'unsigned', 'char'):
+                result = model.UnknownIntegerType(decl.name)
+            elif t in ('float', 'double'):
+                result = model.UnknownFloatType(decl.name)
+            else:
                 raise api.FFIError(':%d: bad usage of "..."' % decl.coord.line)
+
         if self._uses_new_feature is None:
             self._uses_new_feature = "'typedef %s... %s'" % (
                 ' '.join(typenames[:-1]), decl.name)
-        return model.UnknownIntegerType(decl.name)
+
+        return result
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -35,6 +35,9 @@
     def is_integer_type(self):
         return False
 
+    def is_float_type(self):
+        return False
+
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -164,6 +167,20 @@
         raise NotImplementedError("integer type '%s' can only be used after "
                                   "compilation" % self.name)
 
+class UnknownFloatType(BasePrimitiveType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def is_float_type(self):
+        return True  # for now
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("float type '%s' can only be used after "
+                                  "compilation" % self.name)
+
 
 class BaseFunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -468,6 +468,8 @@
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
+            elif tp.is_float_type():
+                converter = '_cffi_to_c_float'
             else:
                 converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
                                                    tp.name.replace(' ', '_'))
@@ -522,6 +524,8 @@
         if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type():
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
+            elif tp.is_float_type():
+                return '_cffi_from_c_double(%s)' % (var, )
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
@@ -1107,6 +1111,10 @@
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
+    def _emit_bytecode_UnknownFloatType(self, tp, index):
+        s = '_cffi_prim_float(sizeof(%s))' % (tp.name, )
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
+
     def _emit_bytecode_RawFunctionType(self, tp, index):
         self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
         index += 1
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1056,7 +1056,6 @@
     assert lib.nu == 20
 
 def test_some_float_type():
-    py.test.skip("later")
     ffi = FFI()
     ffi.cdef("typedef double... foo_t; foo_t sum(foo_t[]);")
     lib = verify(ffi, 'test_some_float_type', """

From noreply at buildbot.pypy.org  Fri Aug 28 10:46:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 10:46:59 +0200 (CEST)
Subject: [pypy-commit] cffi add-float-dotdotdot: Test the various cases,
 fix issues with rounding, etc.
Message-ID: <20150828084659.8376F1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: add-float-dotdotdot
Changeset: r2260:349afe35982d
Date: 2015-08-28 10:47 +0200
http://bitbucket.org/cffi/cffi/changeset/349afe35982d/

Log:	Test the various cases, fix issues with rounding, etc.

	A subtle point is that your version uses is_float_type(), which is
	true also for "long double", and so it will loose precision on
	platforms like x86 Linux where "long double" is larger than
	"double".

diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -166,6 +166,17 @@
                         "size (or not an integer type at all)");
         return NULL;
     }
+    else if (num == _CFFI__UNKNOWN_FLOAT_PRIM) {
+        PyErr_SetString(FFIError, "primitive floating-point type with an "
+                        "unexpected size (or not a float type at all)");
+        return NULL;
+    }
+    else if (num == _CFFI__UNKNOWN_LONG_DOUBLE) {
+        PyErr_SetString(FFIError, "primitive floating-point type is "
+                        "'long double', not supported for now with "
+                        "the syntax 'typedef double... xxx;'");
+        return NULL;
+    }
     else {
         PyErr_Format(PyExc_NotImplementedError, "prim=%d", num);
         return NULL;
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -215,9 +215,10 @@
      _CFFI__UNKNOWN_PRIM)
 
 #define _cffi_prim_float(size)                                          \
-    ((size) == 4 ? _CFFI_PRIM_FLOAT :                                   \
-     (size) == 8 ? _CFFI_PRIM_DOUBLE :                                  \
-     _CFFI__UNKNOWN_PRIM)
+    ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT :                       \
+     (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE :                     \
+     (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE :       \
+     _CFFI__UNKNOWN_FLOAT_PRIM)
 
 #define _cffi_check_int(got, got_nonpos, expected)      \
     ((got_nonpos) == (expected <= 0) &&                 \
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -106,7 +106,9 @@
 PRIM_UINTMAX       = 47
 
 _NUM_PRIM          = 48
-_UNKNOWN_PRIM      = -1
+_UNKNOWN_PRIM          = -1
+_UNKNOWN_FLOAT_PRIM    = -2
+_UNKNOWN_LONG_DOUBLE   = -3
 
 PRIMITIVE_TO_INDEX = {
     'char':               PRIM_CHAR,
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -649,14 +649,17 @@
         if len(typenames) == 1:
             return model.unknown_type(decl.name)
 
-        result = None
-        for t in typenames[:-1]:
-            if t in ('int', 'short', 'long', 'signed', 'unsigned', 'char'):
-                result = model.UnknownIntegerType(decl.name)
-            elif t in ('float', 'double'):
-                result = model.UnknownFloatType(decl.name)
-            else:
-                raise api.FFIError(':%d: bad usage of "..."' % decl.coord.line)
+        if (typenames[:-1] == ['float'] or
+            typenames[:-1] == ['double']):
+            # not for 'long double' so far
+            result = model.UnknownFloatType(decl.name)
+        else:
+            for t in typenames[:-1]:
+                if t not in ['int', 'short', 'long', 'signed',
+                             'unsigned', 'char']:
+                    raise api.FFIError(':%d: bad usage of "..."' %
+                                       decl.coord.line)
+            result = model.UnknownIntegerType(decl.name)
 
         if self._uses_new_feature is None:
             self._uses_new_feature = "'typedef %s... %s'" % (
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def is_float_type(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -161,7 +158,7 @@
         self.c_name_with_marker = name + '&'
 
     def is_integer_type(self):
-        return True    # for now
+        return True
 
     def build_backend_type(self, ffi, finishlist):
         raise NotImplementedError("integer type '%s' can only be used after "
@@ -174,9 +171,6 @@
         self.name = name
         self.c_name_with_marker = name + '&'
 
-    def is_float_type(self):
-        return True  # for now
-
     def build_backend_type(self, ffi, finishlist):
         raise NotImplementedError("float type '%s' can only be used after "
                                   "compilation" % self.name)
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -79,7 +79,9 @@
 #define _CFFI_PRIM_UINTMAX      47
 
 #define _CFFI__NUM_PRIM         48
-#define _CFFI__UNKNOWN_PRIM    (-1)
+#define _CFFI__UNKNOWN_PRIM           (-1)
+#define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
+#define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
 
 
 struct _cffi_global_s {
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -468,8 +468,10 @@
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
-            elif tp.is_float_type():
-                converter = '_cffi_to_c_float'
+            elif isinstance(tp, model.UnknownFloatType):
+                # don't check with is_float_type(): it may be a 'long
+                # double' here, and _cffi_to_c_double would loose precision
+                converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
                 converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
                                                    tp.name.replace(' ', '_'))
@@ -524,8 +526,8 @@
         if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type():
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
-            elif tp.is_float_type():
-                return '_cffi_from_c_double(%s)' % (var, )
+            elif isinstance(tp, model.UnknownFloatType):
+                return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
@@ -1112,7 +1114,9 @@
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
     def _emit_bytecode_UnknownFloatType(self, tp, index):
-        s = '_cffi_prim_float(sizeof(%s))' % (tp.name, )
+        s = ('_cffi_prim_float(sizeof(%s) *\n'
+             '           (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n'
+             '         )' % (tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
     def _emit_bytecode_RawFunctionType(self, tp, index):
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1,6 +1,6 @@
 
 import sys, os, py
-from cffi import FFI, VerificationError
+from cffi import FFI, VerificationError, FFIError
 from cffi import recompiler
 from testing.udir import udir
 from testing.support import u
@@ -1057,12 +1057,51 @@
 
 def test_some_float_type():
     ffi = FFI()
-    ffi.cdef("typedef double... foo_t; foo_t sum(foo_t[]);")
+    ffi.cdef("""
+        typedef double... foo_t;
+        typedef float... bar_t;
+        foo_t sum(foo_t[]);
+        bar_t neg(bar_t);
+        """)
     lib = verify(ffi, 'test_some_float_type', """
         typedef float foo_t;
         static foo_t sum(foo_t x[]) { return x[0] + x[1]; }
+        typedef double bar_t;
+        static double neg(double x) { return -x; }
     """)
     assert lib.sum([40.0, 2.25]) == 42.25
+    assert lib.sum([12.3, 45.6]) != 12.3 + 45.6     # precision loss
+    assert lib.neg(12.3) == -12.3                   # no precision loss
+
+def test_some_float_invalid_1():
+    ffi = FFI()
+    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+
+def test_some_float_invalid_2():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_2', """
+        typedef unsigned long foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    e = py.test.raises(ffi.error, getattr, lib, 'neg')
+    assert str(e.value) == ("primitive floating-point type with an unexpected "
+                            "size (or not a float type at all)")
+
+def test_some_float_invalid_3():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_3', """
+        typedef long double foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    if ffi.sizeof("long double") == ffi.sizeof("double"):
+        assert lib.neg(12.3) == -12.3
+    else:
+        e = py.test.raises(ffi.error, getattr, lib, 'neg')
+        assert str(e.value) == ("primitive floating-point type is "
+                                "'long double', not supported for now with "
+                                "the syntax 'typedef double... xxx;'")
 
 def test_issue200():
     ffi = FFI()

From noreply at buildbot.pypy.org  Fri Aug 28 10:52:14 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 10:52:14 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: silence a prevalent rpython
	warning
Message-ID: <20150828085214.8B67E1C0400@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79266:fb11a2c775e9
Date: 2015-08-28 10:50 +0200
http://bitbucket.org/pypy/pypy/changeset/fb11a2c775e9/

Log:	silence a prevalent rpython warning

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -226,6 +226,7 @@
         (_, _, ofs) = symbolic.get_array_token(ARRAY_OR_STRUCT, tsc)
         size = symbolic.get_size(lltype.Signed, tsc)
         result = FieldDescr("len", ofs, size, get_type_flag(lltype.Signed))
+        result.parent_descr = None
         cache[ARRAY_OR_STRUCT] = result
         return result
 

From noreply at buildbot.pypy.org  Fri Aug 28 11:00:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 11:00:32 +0200 (CEST)
Subject: [pypy-commit] cffi add-float-dotdotdot: Document
Message-ID: <20150828090032.1BA491C0400@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: add-float-dotdotdot
Changeset: r2261:f00569dace33
Date: 2015-08-28 11:01 +0200
http://bitbucket.org/cffi/cffi/changeset/f00569dace33/

Log:	Document

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -394,6 +394,16 @@
    ``(u)int(8,16,32,64)_t`` in Python, but in the generated C code,
    only ``foo_t`` is used.
 
+* *New in version 1.3:* floating-point types: "``typedef
+  float... foo_t;``" (or equivalently "``typedef double... foo_t;``")
+  declares ``foo_t`` as a-float-or-a-double; the compiler will figure
+  out which it is.  Note that if the actual C type is even larger
+  (``long double`` on some platforms), then compilation will fail.
+  The problem is that the Python "float" type cannot be used to store
+  the extra precision.  (Use the non-dot-dot-dot syntax ``typedef long
+  double foo_t;`` as usual, which returns values that are not Python
+  floats at all but cdata "long double" objects.)
+
 *  unknown types: the syntax "``typedef ... foo_t;``" declares the type
    ``foo_t`` as opaque.  Useful mainly for when the API takes and returns
    ``foo_t *`` without you needing to look inside the ``foo_t``.  Also
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,12 @@
 ======================
 
 
-v1.2.2
-======
+v1.3
+====
+
+* Pull request #64: out-of-line API mode: we can now declare
+  floating-point types with ``typedef float... foo_t;``.  This only
+  works if ``foo_t`` is a float or a double, not ``long double``.
 
 * Issue #217: fix possible unaligned pointer manipulation, which crash
   on some architectures (64-bit, non-x86).

From noreply at buildbot.pypy.org  Fri Aug 28 11:03:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 11:03:27 +0200 (CEST)
Subject: [pypy-commit] cffi add-float-dotdotdot: Extra test
Message-ID: <20150828090327.BCDF61C0400@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: add-float-dotdotdot
Changeset: r2262:99522540de8d
Date: 2015-08-28 11:04 +0200
http://bitbucket.org/cffi/cffi/changeset/99522540de8d/

Log:	Extra test

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1072,6 +1072,8 @@
     assert lib.sum([40.0, 2.25]) == 42.25
     assert lib.sum([12.3, 45.6]) != 12.3 + 45.6     # precision loss
     assert lib.neg(12.3) == -12.3                   # no precision loss
+    assert ffi.sizeof("foo_t") == ffi.sizeof("float")
+    assert ffi.sizeof("bar_t") == ffi.sizeof("double")
 
 def test_some_float_invalid_1():
     ffi = FFI()

From noreply at buildbot.pypy.org  Fri Aug 28 11:10:59 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 11:10:59 +0200 (CEST)
Subject: [pypy-commit] cffi add-float-dotdotdot: Close branch, ready to merge
Message-ID: <20150828091059.22B231C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: add-float-dotdotdot
Changeset: r2263:58516fa7b41a
Date: 2015-08-28 11:11 +0200
http://bitbucket.org/cffi/cffi/changeset/58516fa7b41a/

Log:	Close branch, ready to merge


From noreply at buildbot.pypy.org  Fri Aug 28 11:11:01 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 11:11:01 +0200 (CEST)
Subject: [pypy-commit] cffi default: hg merge add-float-dotdotdot
Message-ID: <20150828091101.296EC1C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2264:6a79f6cdd2b8
Date: 2015-08-28 11:11 +0200
http://bitbucket.org/cffi/cffi/changeset/6a79f6cdd2b8/

Log:	hg merge add-float-dotdotdot

diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -166,6 +166,17 @@
                         "size (or not an integer type at all)");
         return NULL;
     }
+    else if (num == _CFFI__UNKNOWN_FLOAT_PRIM) {
+        PyErr_SetString(FFIError, "primitive floating-point type with an "
+                        "unexpected size (or not a float type at all)");
+        return NULL;
+    }
+    else if (num == _CFFI__UNKNOWN_LONG_DOUBLE) {
+        PyErr_SetString(FFIError, "primitive floating-point type is "
+                        "'long double', not supported for now with "
+                        "the syntax 'typedef double... xxx;'");
+        return NULL;
+    }
     else {
         PyErr_Format(PyExc_NotImplementedError, "prim=%d", num);
         return NULL;
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -214,6 +214,12 @@
      (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) :    \
      _CFFI__UNKNOWN_PRIM)
 
+#define _cffi_prim_float(size)                                          \
+    ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT :                       \
+     (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE :                     \
+     (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE :       \
+     _CFFI__UNKNOWN_FLOAT_PRIM)
+
 #define _cffi_check_int(got, got_nonpos, expected)      \
     ((got_nonpos) == (expected <= 0) &&                 \
      (got) == (unsigned long long)expected)
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -106,7 +106,9 @@
 PRIM_UINTMAX       = 47
 
 _NUM_PRIM          = 48
-_UNKNOWN_PRIM      = -1
+_UNKNOWN_PRIM          = -1
+_UNKNOWN_FLOAT_PRIM    = -2
+_UNKNOWN_LONG_DOUBLE   = -3
 
 PRIMITIVE_TO_INDEX = {
     'char':               PRIM_CHAR,
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -648,10 +648,21 @@
         assert typenames[-1] == '__dotdotdot__'
         if len(typenames) == 1:
             return model.unknown_type(decl.name)
-        for t in typenames[:-1]:
-            if t not in ['int', 'short', 'long', 'signed', 'unsigned', 'char']:
-                raise api.FFIError(':%d: bad usage of "..."' % decl.coord.line)
+
+        if (typenames[:-1] == ['float'] or
+            typenames[:-1] == ['double']):
+            # not for 'long double' so far
+            result = model.UnknownFloatType(decl.name)
+        else:
+            for t in typenames[:-1]:
+                if t not in ['int', 'short', 'long', 'signed',
+                             'unsigned', 'char']:
+                    raise api.FFIError(':%d: bad usage of "..."' %
+                                       decl.coord.line)
+            result = model.UnknownIntegerType(decl.name)
+
         if self._uses_new_feature is None:
             self._uses_new_feature = "'typedef %s... %s'" % (
                 ' '.join(typenames[:-1]), decl.name)
-        return model.UnknownIntegerType(decl.name)
+
+        return result
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -158,12 +158,23 @@
         self.c_name_with_marker = name + '&'
 
     def is_integer_type(self):
-        return True    # for now
+        return True
 
     def build_backend_type(self, ffi, finishlist):
         raise NotImplementedError("integer type '%s' can only be used after "
                                   "compilation" % self.name)
 
+class UnknownFloatType(BasePrimitiveType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("float type '%s' can only be used after "
+                                  "compilation" % self.name)
+
 
 class BaseFunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -79,7 +79,9 @@
 #define _CFFI_PRIM_UINTMAX      47
 
 #define _CFFI__NUM_PRIM         48
-#define _CFFI__UNKNOWN_PRIM    (-1)
+#define _CFFI__UNKNOWN_PRIM           (-1)
+#define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
+#define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
 
 
 struct _cffi_global_s {
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -468,6 +468,10 @@
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
+            elif isinstance(tp, model.UnknownFloatType):
+                # don't check with is_float_type(): it may be a 'long
+                # double' here, and _cffi_to_c_double would loose precision
+                converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
                 converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
                                                    tp.name.replace(' ', '_'))
@@ -522,6 +526,8 @@
         if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type():
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
+            elif isinstance(tp, model.UnknownFloatType):
+                return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
@@ -1107,6 +1113,12 @@
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
+    def _emit_bytecode_UnknownFloatType(self, tp, index):
+        s = ('_cffi_prim_float(sizeof(%s) *\n'
+             '           (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n'
+             '         )' % (tp.name, tp.name))
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
+
     def _emit_bytecode_RawFunctionType(self, tp, index):
         self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
         index += 1
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -394,6 +394,16 @@
    ``(u)int(8,16,32,64)_t`` in Python, but in the generated C code,
    only ``foo_t`` is used.
 
+* *New in version 1.3:* floating-point types: "``typedef
+  float... foo_t;``" (or equivalently "``typedef double... foo_t;``")
+  declares ``foo_t`` as a-float-or-a-double; the compiler will figure
+  out which it is.  Note that if the actual C type is even larger
+  (``long double`` on some platforms), then compilation will fail.
+  The problem is that the Python "float" type cannot be used to store
+  the extra precision.  (Use the non-dot-dot-dot syntax ``typedef long
+  double foo_t;`` as usual, which returns values that are not Python
+  floats at all but cdata "long double" objects.)
+
 *  unknown types: the syntax "``typedef ... foo_t;``" declares the type
    ``foo_t`` as opaque.  Useful mainly for when the API takes and returns
    ``foo_t *`` without you needing to look inside the ``foo_t``.  Also
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,12 @@
 ======================
 
 
-v1.2.2
-======
+v1.3
+====
+
+* Pull request #64: out-of-line API mode: we can now declare
+  floating-point types with ``typedef float... foo_t;``.  This only
+  works if ``foo_t`` is a float or a double, not ``long double``.
 
 * Issue #217: fix possible unaligned pointer manipulation, which crash
   on some architectures (64-bit, non-x86).
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1,6 +1,6 @@
 
 import sys, os, py
-from cffi import FFI, VerificationError
+from cffi import FFI, VerificationError, FFIError
 from cffi import recompiler
 from testing.udir import udir
 from testing.support import u
@@ -1056,14 +1056,54 @@
     assert lib.nu == 20
 
 def test_some_float_type():
-    py.test.skip("later")
     ffi = FFI()
-    ffi.cdef("typedef double... foo_t; foo_t sum(foo_t[]);")
+    ffi.cdef("""
+        typedef double... foo_t;
+        typedef float... bar_t;
+        foo_t sum(foo_t[]);
+        bar_t neg(bar_t);
+        """)
     lib = verify(ffi, 'test_some_float_type', """
         typedef float foo_t;
         static foo_t sum(foo_t x[]) { return x[0] + x[1]; }
+        typedef double bar_t;
+        static double neg(double x) { return -x; }
     """)
     assert lib.sum([40.0, 2.25]) == 42.25
+    assert lib.sum([12.3, 45.6]) != 12.3 + 45.6     # precision loss
+    assert lib.neg(12.3) == -12.3                   # no precision loss
+    assert ffi.sizeof("foo_t") == ffi.sizeof("float")
+    assert ffi.sizeof("bar_t") == ffi.sizeof("double")
+
+def test_some_float_invalid_1():
+    ffi = FFI()
+    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+
+def test_some_float_invalid_2():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_2', """
+        typedef unsigned long foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    e = py.test.raises(ffi.error, getattr, lib, 'neg')
+    assert str(e.value) == ("primitive floating-point type with an unexpected "
+                            "size (or not a float type at all)")
+
+def test_some_float_invalid_3():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_3', """
+        typedef long double foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    if ffi.sizeof("long double") == ffi.sizeof("double"):
+        assert lib.neg(12.3) == -12.3
+    else:
+        e = py.test.raises(ffi.error, getattr, lib, 'neg')
+        assert str(e.value) == ("primitive floating-point type is "
+                                "'long double', not supported for now with "
+                                "the syntax 'typedef double... xxx;'")
 
 def test_issue200():
     ffi = FFI()

From noreply at buildbot.pypy.org  Fri Aug 28 11:19:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 11:19:05 +0200 (CEST)
Subject: [pypy-commit] cffi default: Prepare release 1.3.0
Message-ID: <20150828091905.DE3711C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2265:03ccc967c61c
Date: 2015-08-28 11:19 +0200
http://bitbucket.org/cffi/cffi/changeset/03ccc967c61c/

Log:	Prepare release 1.3.0

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6296,7 +6296,7 @@
     if (v == NULL || PyModule_AddObject(m, "_C_API", v) < 0)
         INITERROR;
 
-    v = PyText_FromString("1.2.1");
+    v = PyText_FromString("1.3.0");
     if (v == NULL || PyModule_AddObject(m, "__version__", v) < 0)
         INITERROR;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.2.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.3.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.1"
-__version_info__ = (1, 2, 1)
+__version__ = "1.3.0"
+__version_info__ = (1, 3, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.2'
+version = '1.3'
 # The full version, including alpha/beta/rc tags.
-release = '1.2.1'
+release = '1.3.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,11 +51,11 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.2.1.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.3.0.tar.gz
 
-   - MD5: faca1e420e80433db409bb1bdd0a3f45
+   - MD5: ...
 
-   - SHA: f7ed014ff1602a8e81073f5356b4cafbc3f5dce1
+   - SHA: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,8 +3,8 @@
 ======================
 
 
-v1.3
-====
+v1.3.0
+======
 
 * Pull request #64: out-of-line API mode: we can now declare
   floating-point types with ``typedef float... foo_t;``.  This only
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@
 
 `Mailing list `_
 """,
-        version='1.2.1',
+        version='1.3.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
                      if cpython else {},

From noreply at buildbot.pypy.org  Fri Aug 28 11:28:05 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 11:28:05 +0200 (CEST)
Subject: [pypy-commit] pypy default: Update to cffi 1.3.0
Message-ID: <20150828092805.D012C1C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79267:5345333d8dcd
Date: 2015-08-28 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/5345333d8dcd/

Log:	Update to cffi 1.3.0

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.1
+Version: 1.3.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.1"
-__version_info__ = (1, 2, 1)
+__version__ = "1.3.0"
+__version_info__ = (1, 3, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -214,6 +214,12 @@
      (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) :    \
      _CFFI__UNKNOWN_PRIM)
 
+#define _cffi_prim_float(size)                                          \
+    ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT :                       \
+     (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE :                     \
+     (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE :       \
+     _CFFI__UNKNOWN_FLOAT_PRIM)
+
 #define _cffi_check_int(got, got_nonpos, expected)      \
     ((got_nonpos) == (expected <= 0) &&                 \
      (got) == (unsigned long long)expected)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -106,7 +106,9 @@
 PRIM_UINTMAX       = 47
 
 _NUM_PRIM          = 48
-_UNKNOWN_PRIM      = -1
+_UNKNOWN_PRIM          = -1
+_UNKNOWN_FLOAT_PRIM    = -2
+_UNKNOWN_LONG_DOUBLE   = -3
 
 PRIMITIVE_TO_INDEX = {
     'char':               PRIM_CHAR,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -648,10 +648,21 @@
         assert typenames[-1] == '__dotdotdot__'
         if len(typenames) == 1:
             return model.unknown_type(decl.name)
-        for t in typenames[:-1]:
-            if t not in ['int', 'short', 'long', 'signed', 'unsigned', 'char']:
-                raise api.FFIError(':%d: bad usage of "..."' % decl.coord.line)
+
+        if (typenames[:-1] == ['float'] or
+            typenames[:-1] == ['double']):
+            # not for 'long double' so far
+            result = model.UnknownFloatType(decl.name)
+        else:
+            for t in typenames[:-1]:
+                if t not in ['int', 'short', 'long', 'signed',
+                             'unsigned', 'char']:
+                    raise api.FFIError(':%d: bad usage of "..."' %
+                                       decl.coord.line)
+            result = model.UnknownIntegerType(decl.name)
+
         if self._uses_new_feature is None:
             self._uses_new_feature = "'typedef %s... %s'" % (
                 ' '.join(typenames[:-1]), decl.name)
-        return model.UnknownIntegerType(decl.name)
+
+        return result
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -158,12 +158,23 @@
         self.c_name_with_marker = name + '&'
 
     def is_integer_type(self):
-        return True    # for now
+        return True
 
     def build_backend_type(self, ffi, finishlist):
         raise NotImplementedError("integer type '%s' can only be used after "
                                   "compilation" % self.name)
 
+class UnknownFloatType(BasePrimitiveType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("float type '%s' can only be used after "
+                                  "compilation" % self.name)
+
 
 class BaseFunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,7 +79,9 @@
 #define _CFFI_PRIM_UINTMAX      47
 
 #define _CFFI__NUM_PRIM         48
-#define _CFFI__UNKNOWN_PRIM    (-1)
+#define _CFFI__UNKNOWN_PRIM           (-1)
+#define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
+#define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
 
 
 struct _cffi_global_s {
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -468,6 +468,10 @@
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
+            elif isinstance(tp, model.UnknownFloatType):
+                # don't check with is_float_type(): it may be a 'long
+                # double' here, and _cffi_to_c_double would loose precision
+                converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
                 converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
                                                    tp.name.replace(' ', '_'))
@@ -522,6 +526,8 @@
         if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type():
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
+            elif isinstance(tp, model.UnknownFloatType):
+                return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
@@ -1107,6 +1113,12 @@
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
+    def _emit_bytecode_UnknownFloatType(self, tp, index):
+        s = ('_cffi_prim_float(sizeof(%s) *\n'
+             '           (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n'
+             '         )' % (tp.name, tp.name))
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
+
     def _emit_bytecode_RawFunctionType(self, tp, index):
         self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
         index += 1
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.1"
+VERSION = "1.3.0"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -9,16 +9,16 @@
             assert isinstance(self.arg, str)
             return '(_cffi_opcode_t)(%s)' % (self.arg,)
         classname = CLASS_NAME[self.op]
-        return '_CFFI_OP(_CFFI_OP_%s, %d)' % (classname, self.arg)
+        return '_CFFI_OP(_CFFI_OP_%s, %s)' % (classname, self.arg)
 
     def as_python_bytes(self):
-        if self.op is None:
-            if self.arg.isdigit():
-                value = int(self.arg)     # non-negative: '-' not in self.arg
-                if value >= 2**31:
-                    raise OverflowError("cannot emit %r: limited to 2**31-1"
-                                        % (self.arg,))
-                return format_four_bytes(value)
+        if self.op is None and self.arg.isdigit():
+            value = int(self.arg)     # non-negative: '-' not in self.arg
+            if value >= 2**31:
+                raise OverflowError("cannot emit %r: limited to 2**31-1"
+                                    % (self.arg,))
+            return format_four_bytes(value)
+        if isinstance(self.arg, str):
             from .ffiplatform import VerificationError
             raise VerificationError("cannot emit to Python: %r" % (self.arg,))
         return format_four_bytes((self.arg << 8) | self.op)
@@ -106,7 +106,9 @@
 PRIM_UINTMAX       = 47
 
 _NUM_PRIM          = 48
-_UNKNOWN_PRIM      = -1
+_UNKNOWN_PRIM          = -1
+_UNKNOWN_FLOAT_PRIM    = -2
+_UNKNOWN_LONG_DOUBLE   = -3
 
 PRIMITIVE_TO_INDEX = {
     'char':               PRIM_CHAR,
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -81,6 +81,13 @@
         if num == cffi_opcode._UNKNOWN_PRIM:
             raise oefmt(ffi.w_FFIError, "primitive integer type with an "
                         "unexpected size (or not an integer type at all)")
+        elif num == cffi_opcode._UNKNOWN_FLOAT_PRIM:
+            raise oefmt(ffi.w_FFIError, "primitive floating-point type with an "
+                        "unexpected size (or not a float type at all)")
+        elif num == cffi_opcode._UNKNOWN_LONG_DOUBLE:
+            raise oefmt(ffi.w_FFIError, "primitive floating-point type is "
+                        "'long double', not supported for now with "
+                        "the syntax 'typedef double... xxx;'")
         else:
             raise oefmt(space.w_NotImplementedError, "prim=%d", num)
     realize_cache = space.fromcache(RealizeCache)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.2.1", ("This test_c.py file is for testing a version"
+assert __version__ == "1.3.0", ("This test_c.py file is for testing a version"
                                 " of cffi that differs from the one that we"
                                 " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1,7 +1,7 @@
 # Generated by pypy/tool/import_cffi.py
 
 import sys, os, py
-from cffi import FFI, VerificationError
+from cffi import FFI, VerificationError, FFIError
 from cffi import recompiler
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 from pypy.module.test_lib_pypy.cffi_tests.support import u
@@ -1057,14 +1057,54 @@
     assert lib.nu == 20
 
 def test_some_float_type():
-    py.test.skip("later")
     ffi = FFI()
-    ffi.cdef("typedef double... foo_t; foo_t sum(foo_t[]);")
+    ffi.cdef("""
+        typedef double... foo_t;
+        typedef float... bar_t;
+        foo_t sum(foo_t[]);
+        bar_t neg(bar_t);
+        """)
     lib = verify(ffi, 'test_some_float_type', """
         typedef float foo_t;
         static foo_t sum(foo_t x[]) { return x[0] + x[1]; }
+        typedef double bar_t;
+        static double neg(double x) { return -x; }
     """)
     assert lib.sum([40.0, 2.25]) == 42.25
+    assert lib.sum([12.3, 45.6]) != 12.3 + 45.6     # precision loss
+    assert lib.neg(12.3) == -12.3                   # no precision loss
+    assert ffi.sizeof("foo_t") == ffi.sizeof("float")
+    assert ffi.sizeof("bar_t") == ffi.sizeof("double")
+
+def test_some_float_invalid_1():
+    ffi = FFI()
+    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+
+def test_some_float_invalid_2():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_2', """
+        typedef unsigned long foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    e = py.test.raises(ffi.error, getattr, lib, 'neg')
+    assert str(e.value) == ("primitive floating-point type with an unexpected "
+                            "size (or not a float type at all)")
+
+def test_some_float_invalid_3():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_3', """
+        typedef long double foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    if ffi.sizeof("long double") == ffi.sizeof("double"):
+        assert lib.neg(12.3) == -12.3
+    else:
+        e = py.test.raises(ffi.error, getattr, lib, 'neg')
+        assert str(e.value) == ("primitive floating-point type is "
+                                "'long double', not supported for now with "
+                                "the syntax 'typedef double... xxx;'")
 
 def test_issue200():
     ffi = FFI()

From noreply at buildbot.pypy.org  Fri Aug 28 11:39:41 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Aug 2015 11:39:41 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: merge default into release for
	cffi 1.3
Message-ID: <20150828093941.8ECAF1C01F4@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79268:00d367b4253b
Date: 2015-08-28 12:38 +0300
http://bitbucket.org/pypy/pypy/changeset/00d367b4253b/

Log:	merge default into release for cffi 1.3

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.2.1
+Version: 1.3.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "1.2.1"
-__version_info__ = (1, 2, 1)
+__version__ = "1.3.0"
+__version_info__ = (1, 3, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -214,6 +214,12 @@
      (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) :    \
      _CFFI__UNKNOWN_PRIM)
 
+#define _cffi_prim_float(size)                                          \
+    ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT :                       \
+     (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE :                     \
+     (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE :       \
+     _CFFI__UNKNOWN_FLOAT_PRIM)
+
 #define _cffi_check_int(got, got_nonpos, expected)      \
     ((got_nonpos) == (expected <= 0) &&                 \
      (got) == (unsigned long long)expected)
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -106,7 +106,9 @@
 PRIM_UINTMAX       = 47
 
 _NUM_PRIM          = 48
-_UNKNOWN_PRIM      = -1
+_UNKNOWN_PRIM          = -1
+_UNKNOWN_FLOAT_PRIM    = -2
+_UNKNOWN_LONG_DOUBLE   = -3
 
 PRIMITIVE_TO_INDEX = {
     'char':               PRIM_CHAR,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -648,10 +648,21 @@
         assert typenames[-1] == '__dotdotdot__'
         if len(typenames) == 1:
             return model.unknown_type(decl.name)
-        for t in typenames[:-1]:
-            if t not in ['int', 'short', 'long', 'signed', 'unsigned', 'char']:
-                raise api.FFIError(':%d: bad usage of "..."' % decl.coord.line)
+
+        if (typenames[:-1] == ['float'] or
+            typenames[:-1] == ['double']):
+            # not for 'long double' so far
+            result = model.UnknownFloatType(decl.name)
+        else:
+            for t in typenames[:-1]:
+                if t not in ['int', 'short', 'long', 'signed',
+                             'unsigned', 'char']:
+                    raise api.FFIError(':%d: bad usage of "..."' %
+                                       decl.coord.line)
+            result = model.UnknownIntegerType(decl.name)
+
         if self._uses_new_feature is None:
             self._uses_new_feature = "'typedef %s... %s'" % (
                 ' '.join(typenames[:-1]), decl.name)
-        return model.UnknownIntegerType(decl.name)
+
+        return result
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -158,12 +158,23 @@
         self.c_name_with_marker = name + '&'
 
     def is_integer_type(self):
-        return True    # for now
+        return True
 
     def build_backend_type(self, ffi, finishlist):
         raise NotImplementedError("integer type '%s' can only be used after "
                                   "compilation" % self.name)
 
+class UnknownFloatType(BasePrimitiveType):
+    _attrs_ = ('name', )
+
+    def __init__(self, name):
+        self.name = name
+        self.c_name_with_marker = name + '&'
+
+    def build_backend_type(self, ffi, finishlist):
+        raise NotImplementedError("float type '%s' can only be used after "
+                                  "compilation" % self.name)
+
 
 class BaseFunctionType(BaseType):
     _attrs_ = ('args', 'result', 'ellipsis')
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,7 +79,9 @@
 #define _CFFI_PRIM_UINTMAX      47
 
 #define _CFFI__NUM_PRIM         48
-#define _CFFI__UNKNOWN_PRIM    (-1)
+#define _CFFI__UNKNOWN_PRIM           (-1)
+#define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
+#define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
 
 
 struct _cffi_global_s {
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -468,6 +468,10 @@
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
+            elif isinstance(tp, model.UnknownFloatType):
+                # don't check with is_float_type(): it may be a 'long
+                # double' here, and _cffi_to_c_double would loose precision
+                converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
                 converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
                                                    tp.name.replace(' ', '_'))
@@ -522,6 +526,8 @@
         if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type():
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
+            elif isinstance(tp, model.UnknownFloatType):
+                return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
@@ -1107,6 +1113,12 @@
              '         ) <= 0)' % (tp.name, tp.name, tp.name))
         self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
 
+    def _emit_bytecode_UnknownFloatType(self, tp, index):
+        s = ('_cffi_prim_float(sizeof(%s) *\n'
+             '           (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n'
+             '         )' % (tp.name, tp.name))
+        self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s)
+
     def _emit_bytecode_RawFunctionType(self, tp, index):
         self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result])
         index += 1
diff --git a/lib_pypy/ctypes_support.py b/lib_pypy/ctypes_support.py
--- a/lib_pypy/ctypes_support.py
+++ b/lib_pypy/ctypes_support.py
@@ -28,7 +28,7 @@
     def _where_is_errno():
         return standard_c_lib.__errno_location()
 
-elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'):
+elif sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
     standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int)
     standard_c_lib.__error.argtypes = None
     def _where_is_errno():
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.2.1"
+VERSION = "1.3.0"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -9,16 +9,16 @@
             assert isinstance(self.arg, str)
             return '(_cffi_opcode_t)(%s)' % (self.arg,)
         classname = CLASS_NAME[self.op]
-        return '_CFFI_OP(_CFFI_OP_%s, %d)' % (classname, self.arg)
+        return '_CFFI_OP(_CFFI_OP_%s, %s)' % (classname, self.arg)
 
     def as_python_bytes(self):
-        if self.op is None:
-            if self.arg.isdigit():
-                value = int(self.arg)     # non-negative: '-' not in self.arg
-                if value >= 2**31:
-                    raise OverflowError("cannot emit %r: limited to 2**31-1"
-                                        % (self.arg,))
-                return format_four_bytes(value)
+        if self.op is None and self.arg.isdigit():
+            value = int(self.arg)     # non-negative: '-' not in self.arg
+            if value >= 2**31:
+                raise OverflowError("cannot emit %r: limited to 2**31-1"
+                                    % (self.arg,))
+            return format_four_bytes(value)
+        if isinstance(self.arg, str):
             from .ffiplatform import VerificationError
             raise VerificationError("cannot emit to Python: %r" % (self.arg,))
         return format_four_bytes((self.arg << 8) | self.op)
@@ -106,7 +106,9 @@
 PRIM_UINTMAX       = 47
 
 _NUM_PRIM          = 48
-_UNKNOWN_PRIM      = -1
+_UNKNOWN_PRIM          = -1
+_UNKNOWN_FLOAT_PRIM    = -2
+_UNKNOWN_LONG_DOUBLE   = -3
 
 PRIMITIVE_TO_INDEX = {
     'char':               PRIM_CHAR,
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -81,6 +81,13 @@
         if num == cffi_opcode._UNKNOWN_PRIM:
             raise oefmt(ffi.w_FFIError, "primitive integer type with an "
                         "unexpected size (or not an integer type at all)")
+        elif num == cffi_opcode._UNKNOWN_FLOAT_PRIM:
+            raise oefmt(ffi.w_FFIError, "primitive floating-point type with an "
+                        "unexpected size (or not a float type at all)")
+        elif num == cffi_opcode._UNKNOWN_LONG_DOUBLE:
+            raise oefmt(ffi.w_FFIError, "primitive floating-point type is "
+                        "'long double', not supported for now with "
+                        "the syntax 'typedef double... xxx;'")
         else:
             raise oefmt(space.w_NotImplementedError, "prim=%d", num)
     realize_cache = space.fromcache(RealizeCache)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,6 +1,9 @@
 # ____________________________________________________________
 
 import sys
+assert __version__ == "1.3.0", ("This test_c.py file is for testing a version"
+                                " of cffi that differs from the one that we"
+                                " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
     type_or_class = "type"
     mandatory_b_prefix = ''
@@ -3424,7 +3427,3 @@
                             "be 'foo *', but the types are different (check "
                             "that you are not e.g. mixing up different ffi "
                             "instances)")
-
-def test_version():
-    # this test is here mostly for PyPy
-    assert __version__ == "1.2.1"
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1,7 +1,7 @@
 # Generated by pypy/tool/import_cffi.py
 
 import sys, os, py
-from cffi import FFI, VerificationError
+from cffi import FFI, VerificationError, FFIError
 from cffi import recompiler
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 from pypy.module.test_lib_pypy.cffi_tests.support import u
@@ -1057,14 +1057,54 @@
     assert lib.nu == 20
 
 def test_some_float_type():
-    py.test.skip("later")
     ffi = FFI()
-    ffi.cdef("typedef double... foo_t; foo_t sum(foo_t[]);")
+    ffi.cdef("""
+        typedef double... foo_t;
+        typedef float... bar_t;
+        foo_t sum(foo_t[]);
+        bar_t neg(bar_t);
+        """)
     lib = verify(ffi, 'test_some_float_type', """
         typedef float foo_t;
         static foo_t sum(foo_t x[]) { return x[0] + x[1]; }
+        typedef double bar_t;
+        static double neg(double x) { return -x; }
     """)
     assert lib.sum([40.0, 2.25]) == 42.25
+    assert lib.sum([12.3, 45.6]) != 12.3 + 45.6     # precision loss
+    assert lib.neg(12.3) == -12.3                   # no precision loss
+    assert ffi.sizeof("foo_t") == ffi.sizeof("float")
+    assert ffi.sizeof("bar_t") == ffi.sizeof("double")
+
+def test_some_float_invalid_1():
+    ffi = FFI()
+    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+
+def test_some_float_invalid_2():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_2', """
+        typedef unsigned long foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    e = py.test.raises(ffi.error, getattr, lib, 'neg')
+    assert str(e.value) == ("primitive floating-point type with an unexpected "
+                            "size (or not a float type at all)")
+
+def test_some_float_invalid_3():
+    ffi = FFI()
+    ffi.cdef("typedef double... foo_t; foo_t neg(foo_t);")
+    lib = verify(ffi, 'test_some_float_invalid_3', """
+        typedef long double foo_t;
+        foo_t neg(foo_t x) { return -x; }
+    """)
+    if ffi.sizeof("long double") == ffi.sizeof("double"):
+        assert lib.neg(12.3) == -12.3
+    else:
+        e = py.test.raises(ffi.error, getattr, lib, 'neg')
+        assert str(e.value) == ("primitive floating-point type is "
+                                "'long double', not supported for now with "
+                                "the syntax 'typedef double... xxx;'")
 
 def test_issue200():
     ffi = FFI()
diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py
--- a/pypy/module/thread/test/test_lock.py
+++ b/pypy/module/thread/test/test_lock.py
@@ -116,9 +116,6 @@
 class AppTestLockSignals(GenericTestThread):
     pytestmark = py.test.mark.skipif("os.name != 'posix'")
 
-    def setup_class(cls):
-        cls.w_using_pthread_cond = cls.space.wrap(sys.platform == 'freebsd6')
-
     def w_acquire_retries_on_intr(self, lock):
         import thread, os, signal, time
         self.sig_recvd = False
@@ -157,8 +154,6 @@
         raise KeyboardInterrupt
 
     def test_lock_acquire_interruption(self):
-        if self.using_pthread_cond:
-            skip('POSIX condition variables cannot be interrupted')
         import thread, signal, time
         # Mimic receiving a SIGINT (KeyboardInterrupt) with SIGALRM while stuck
         # in a deadlock.
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8445,6 +8445,7 @@
         self.optimize_loop(ops, expected)
 
     def test_issue1080_infinitie_loop_virtual(self):
+        # Same comment as the following test_issue1080_infinitie_loop_simple
         ops = """
         [p10]
         p52 = getfield_gc(p10, descr=nextdescr) # inst_storage
@@ -8467,6 +8468,10 @@
         self.raises(InvalidLoop, self.optimize_loop, ops, ops)
 
     def test_issue1080_infinitie_loop_simple(self):
+        # 'quasiimmutdescr' is a QuasiImmutDescr initialized with the
+        # 'quasibox' as the quasi-immutable instance.  We close the loop
+        # with ConstPtr(myptr), i.e. a different pointer.  The test checks
+        # that the resulting loop is invalid.
         ops = """
         [p69]
         quasiimmut_field(p69, descr=quasiimmutdescr)
diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py
--- a/rpython/rlib/clibffi.py
+++ b/rpython/rlib/clibffi.py
@@ -32,7 +32,6 @@
 _WIN32 = _MSVC or _MINGW
 _WIN64 = _WIN32 and is_emulated_long
 _MAC_OS = platform.name == "darwin"
-_FREEBSD_7 = platform.name == "freebsd7"
 
 _LITTLE_ENDIAN = sys.byteorder == 'little'
 _BIG_ENDIAN = sys.byteorder == 'big'
diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py
--- a/rpython/rtyper/lltypesystem/ll2ctypes.py
+++ b/rpython/rtyper/lltypesystem/ll2ctypes.py
@@ -46,6 +46,7 @@
 
 _POSIX = os.name == "posix"
 _MS_WINDOWS = os.name == "nt"
+_FREEBSD = sys.platform.startswith('freebsd')
 _64BIT = "64bit" in host_platform.architecture()[0]
 
 
@@ -1080,8 +1081,11 @@
             return ctypes.util.find_library('c')
 
     libc_name = get_libc_name()     # Make sure the name is determined during import, not at runtime
+    if _FREEBSD:
+        RTLD_DEFAULT = -2  # see 
+        rtld_default_lib = ctypes.CDLL("RTLD_DEFAULT", handle=RTLD_DEFAULT, **load_library_kwargs)
     # XXX is this always correct???
-    standard_c_lib = ctypes.CDLL(get_libc_name(), **load_library_kwargs)
+    standard_c_lib = ctypes.CDLL(libc_name, **load_library_kwargs)
 
 # ____________________________________________
 
@@ -1174,7 +1178,10 @@
                 not_found.append(libname)
 
     if cfunc is None:
-        cfunc = get_on_lib(standard_c_lib, funcname)
+        if _FREEBSD and funcname in ('dlopen', 'fdlopen', 'dlsym', 'dlfunc', 'dlerror', 'dlclose'):
+            cfunc = get_on_lib(rtld_default_lib, funcname)
+        else:
+            cfunc = get_on_lib(standard_c_lib, funcname)
         # XXX magic: on Windows try to load the function from 'kernel32' too
         if cfunc is None and hasattr(ctypes, 'windll'):
             cfunc = get_on_lib(ctypes.windll.kernel32, funcname)
@@ -1493,18 +1500,17 @@
             _where_is_errno().contents.value = TLS.errno
 
     if ctypes:
-        if sys.platform == 'win32':
+        if _MS_WINDOWS:
             standard_c_lib._errno.restype = ctypes.POINTER(ctypes.c_int)
             def _where_is_errno():
                 return standard_c_lib._errno()
 
-        elif sys.platform.startswith('linux') or sys.platform == 'freebsd6':
+        elif sys.platform.startswith('linux'):
             standard_c_lib.__errno_location.restype = ctypes.POINTER(ctypes.c_int)
             def _where_is_errno():
                 return standard_c_lib.__errno_location()
 
-        elif any(plat in sys.platform
-                 for plat in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9')):
+        elif sys.platform == 'darwin' or _FREEBSD:
             standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int)
             def _where_is_errno():
                 return standard_c_lib.__error()

From noreply at buildbot.pypy.org  Fri Aug 28 11:39:43 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Fri, 28 Aug 2015 11:39:43 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: document cffi 1.3
Message-ID: <20150828093943.BC5F61C01F4@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79269:f3ad1e1e1d62
Date: 2015-08-28 12:39 +0300
http://bitbucket.org/pypy/pypy/changeset/f3ad1e1e1d62/

Log:	document cffi 1.3

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -3,7 +3,7 @@
 ==========
 
 We're pleased to announce PyPy 2.6.1, an update to PyPy 2.6.0 released June 1.
-We have updated stdlib to 2.7.10, `cffi`_ to version 1.2, extended support for
+We have updated stdlib to 2.7.10, `cffi`_ to version 1.3, extended support for
 the new vmprof_ statistical profiler for multiple threads, and increased
 functionality of numpy.
 
@@ -87,7 +87,7 @@
 
 * New features:
 
-  * cffi was updated to version 1.2
+  * cffi was updated to version 1.3
 
   * The python stdlib was updated to 2.7.10 from 2.7.9
 

From noreply at buildbot.pypy.org  Fri Aug 28 11:53:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 11:53:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150828095338.8D7111C0400@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79270:79beb329dbbe
Date: 2015-08-28 11:45 +0200
http://bitbucket.org/pypy/pypy/changeset/79beb329dbbe/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -777,6 +777,9 @@
         if 'call_assembler' in check:
             assert check.pop('call_assembler') == 0
             check['call_assembler_i'] = check['call_assembler_r'] = check['call_assembler_f'] = check['call_assembler_n'] = 0
+        if 'getfield_gc' in check:
+            assert check.pop('getfield_gc') == 0
+            check['getfield_gc_i'] = check['getfield_gc_r'] = check['getfield_gc_f'] = 0
         for loop in self.get_all_loops():
             insns = loop.summary(adding_insns=insns, omit_finish=omit_finish)
         return self._check_insns(insns, expected, check)
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -80,7 +80,7 @@
             return xy.inst_x
         res = self.meta_interp(f, [20])
         assert res == 30
-        self.check_simple_loop(setfield_gc=0, getfield_gc=0)
+        self.check_simple_loop(setfield_gc=0, getfield_gc_i=0)
 
     def test_preexisting_access_2(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy'],
@@ -105,8 +105,8 @@
         assert f(5) == 185
         res = self.meta_interp(f, [5])
         assert res == 185
-        self.check_resops(setfield_gc=0,
-                          getfield_gc=2)  # <= at the header of the loop
+        self.check_resops(setfield_gc=0, getfield_gc_r=1,
+                          getfield_gc_i=1)  # <= at the header of the loop
 
     def test_two_paths_access(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy'],
@@ -128,7 +128,7 @@
             return xy.inst_x
         res = self.meta_interp(f, [18])
         assert res == 10118
-        self.check_resops(setfield_gc=0, getfield_gc=2)
+        self.check_resops(setfield_gc=0, getfield_gc_i=1, getfield_gc_r=1)
 
     def test_synchronize_in_return(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy'],
@@ -152,7 +152,7 @@
             return xy.inst_x
         res = self.meta_interp(f, [18])
         assert res == 10180
-        self.check_resops(setfield_gc=0, getfield_gc=2)
+        self.check_resops(setfield_gc=0, getfield_gc_i=1, getfield_gc_r=1)
 
     def test_synchronize_in_return_2(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy'],
@@ -207,7 +207,7 @@
             return res
         res = self.meta_interp(f, [40])
         assert res == 50 * 4
-        self.check_resops(setfield_gc=0, getfield_gc=4)
+        self.check_resops(setfield_gc=0, getfield_gc_i=2, getfield_gc_r=2)
 
     def test_double_frame(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy', 'other'],
@@ -230,8 +230,9 @@
             return xy.inst_x
         res = self.meta_interp(f, [20])
         assert res == 134
-        self.check_simple_loop(setfield_gc=1, getfield_gc=0, cond_call=1)
-        self.check_resops(setfield_gc=2, getfield_gc=4)
+        self.check_simple_loop(setfield_gc=1, getfield_gc_i=0, cond_call=1,
+                               getfield_gc_r=0)
+        self.check_resops(setfield_gc=2, getfield_gc_i=2, getfield_gc_r=2)
 
     # ------------------------------
 

From noreply at buildbot.pypy.org  Fri Aug 28 11:53:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 11:53:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix virtualizables
Message-ID: <20150828095340.A59511C0400@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79271:6aa509ce12e3
Date: 2015-08-28 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/6aa509ce12e3/

Log:	fix virtualizables

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -229,7 +229,8 @@
     if not we_are_translated():
         loop.check_consistency()
     jitcell_token.target_tokens = [target_token]
-    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
+    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
+                         inputargs)
     record_loop_or_bridge(metainterp_sd, loop)
     return target_token
 
@@ -295,7 +296,8 @@
     if not we_are_translated():
         loop.check_consistency()
     jitcell_token.target_tokens = [start_descr] + jitcell_token.target_tokens
-    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop")
+    send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, "loop",
+                         inputargs)
     record_loop_or_bridge(metainterp_sd, loop)
     return start_descr
 
@@ -346,7 +348,7 @@
     #    loop.quasi_immutable_deps = quasi_immutable_deps
 
     target_token = loop.operations[-1].getdescr()
-    resumekey.compile_and_attach(metainterp, loop)
+    resumekey.compile_and_attach(metainterp, loop, inputargs)
 
     record_loop_or_bridge(metainterp_sd, loop)
     return target_token
@@ -380,7 +382,7 @@
                         for a in op.getfailargs()])
     lst.append(op)
 
-def patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd):
+def patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd, vable):
     # XXX merge with rewriting
     vinfo = jitdriver_sd.virtualizable_info
     extra_ops = []
@@ -398,7 +400,6 @@
         i += 1
     arrayindex = 0
     for descr in vinfo.array_field_descrs:
-        vable = vable_box.getref_base()
         arraylen = vinfo.get_array_length(vable, arrayindex)
         arrayop = ResOperation(rop.GETFIELD_GC_R, [vable_box], descr)
         emit_op(extra_ops, arrayop)
@@ -454,12 +455,14 @@
         if reset_values:
             item.reset_value()
 
-def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type):
+def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type,
+                         orig_inpargs):
     forget_optimization_info(loop.operations)
     forget_optimization_info(loop.inputargs)
     vinfo = jitdriver_sd.virtualizable_info
     if vinfo is not None:
-        patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd)
+        vable = orig_inpargs[jitdriver_sd.index_of_virtualizable].getref_base()
+        patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd, vable)
 
     original_jitcell_token = loop.original_jitcell_token
     globaldata = metainterp_sd.globaldata
@@ -756,7 +759,7 @@
         # incremented at all as long as ST_BUSY_FLAG was set.
         self.status &= ~self.ST_BUSY_FLAG
 
-    def compile_and_attach(self, metainterp, new_loop):
+    def compile_and_attach(self, metainterp, new_loop, orig_inputargs):
         # We managed to create a bridge.  Attach the new operations
         # to the corresponding guard_op and compile from there
         assert metainterp.resumekey_original_loop_token is not None
@@ -943,7 +946,7 @@
     def __init__(self, original_greenkey):
         self.original_greenkey = original_greenkey
 
-    def compile_and_attach(self, metainterp, new_loop):
+    def compile_and_attach(self, metainterp, new_loop, orig_inputargs):
         # We managed to create a bridge going from the interpreter
         # to previously-compiled code.  We keep 'new_loop', which is not
         # a loop at all but ends in a jump to the target loop.  It starts
@@ -953,7 +956,8 @@
         new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
         #propagate_original_jitcell_token(new_loop)
         send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
-                             metainterp_sd, new_loop, "entry bridge")
+                             metainterp_sd, new_loop, "entry bridge",
+                             orig_inputargs)
         # send the new_loop to warmspot.py, to be called directly the next time
         jitdriver_sd.warmstate.attach_procedure_to_interp(
             self.original_greenkey, jitcell_token)
@@ -1014,7 +1018,7 @@
     if info.final():
         new_trace.inputargs = info.inputargs
         target_token = new_trace.operations[-1].getdescr()
-        resumekey.compile_and_attach(metainterp, new_trace)
+        resumekey.compile_and_attach(metainterp, new_trace, inputargs)
         record_loop_or_bridge(metainterp_sd, new_trace)
         return target_token
     new_trace.inputargs = info.renamed_inputargs
diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -283,7 +283,8 @@
         res = self.meta_interp(f, [16])
         assert res == 3001 + 16 * 80
         self.check_simple_loop(setarrayitem_gc=0, setfield_gc=0,
-                               getarrayitem_gc=0, getfield_gc=0)
+            getarrayitem_gc_i=0, getarrayitem_gc_r=0, getfield_gc_i=0,
+            getfield_gc_r=0)
 
     def test_synchronize_arrays_in_return(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy2'],

From noreply at buildbot.pypy.org  Fri Aug 28 12:00:12 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 12:00:12 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: finish porting
	test_virtualizable
Message-ID: <20150828100012.B3DC21C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79272:93c7bd243702
Date: 2015-08-28 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/93c7bd243702/

Log:	finish porting test_virtualizable

diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py
--- a/rpython/jit/metainterp/test/test_virtualizable.py
+++ b/rpython/jit/metainterp/test/test_virtualizable.py
@@ -315,8 +315,9 @@
         assert f(18) == 10360
         res = self.meta_interp(f, [18])
         assert res == 10360
-        self.check_simple_loop(setfield_gc=0, getarrayitem_gc=0,
-                               getfield_gc=0, setarrayitem_gc=0)
+        self.check_simple_loop(setfield_gc=0, getarrayitem_gc_i=0,
+                               getfield_gc_i=0, getfield_gc_r=0,
+                               setarrayitem_gc=0, getarrayitem_gc_r=0)
 
     def test_array_length(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy2'],
@@ -342,8 +343,9 @@
             return xy2.inst_l1[1]
         res = self.meta_interp(f, [18])
         assert res == 2941309 + 18
-        self.check_simple_loop(setfield_gc=0, getarrayitem_gc=0,
-                               arraylen_gc=0, getfield_gc=0)
+        self.check_simple_loop(setfield_gc=0, getarrayitem_gc_i=0,
+                               getarrayitem_gc_r=0, getfield_gc_i=0,
+                               arraylen_gc=0, getfield_gc_r=0)
 
     def test_residual_function(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy2'],
@@ -376,8 +378,9 @@
             return xy2.inst_l1[1]
         res = self.meta_interp(f, [18])
         assert res == 2941309 + 18
-        self.check_simple_loop(call=1, setfield_gc=0, getarrayitem_gc=0,
-                               arraylen_gc=1, getfield_gc=0)
+        self.check_simple_loop(call_r=1, setfield_gc=0, getarrayitem_gc_i=0,
+            getarrayitem_gc_r=0, arraylen_gc=1, getfield_gc_i=0,
+            getfield_gc_r=0)
 
     def test_double_frame_array(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'xy2', 'other'],
@@ -414,7 +417,8 @@
         res = self.meta_interp(f, [20], enable_opts='')
         assert res == expected
         self.check_simple_loop(setarrayitem_gc=1, setfield_gc=0,
-                               getarrayitem_gc=1, arraylen_gc=1, getfield_gc=2)
+                               getarrayitem_gc_i=1, arraylen_gc=1,
+                               getfield_gc_r=2)
 
     # ------------------------------
 
@@ -463,7 +467,8 @@
         assert f(18) == 10360
         res = self.meta_interp(f, [18])
         assert res == 10360
-        self.check_simple_loop(getfield_gc=0, getarrayitem_gc=0,
+        self.check_simple_loop(getfield_gc_i=0, getfield_gc_r=0,
+                               getarrayitem_gc_i=0, getarrayitem_gc_r=0,
                                setfield_gc=0, setarrayitem_gc=0)
 
     # ------------------------------
@@ -497,7 +502,8 @@
 
         res = self.meta_interp(f, [10])
         assert res == 55
-        self.check_simple_loop(setfield_gc=0, getfield_gc=0)
+        self.check_simple_loop(setfield_gc=0, getfield_gc_i=0,
+                               getfield_gc_r=0)
 
     def test_virtualizable_with_array(self):
         myjitdriver = JitDriver(greens = [], reds = ['n', 'x', 'frame'],
@@ -531,8 +537,10 @@
 
         res = self.meta_interp(f, [10, 1], listops=True)
         assert res == f(10, 1)
-        self.check_simple_loop(getfield_gc=0, getarrayitem_gc=0)
-        self.check_resops(getfield_gc=2, getarrayitem_gc=4)
+        self.check_simple_loop(getfield_gc_i=0, getarrayitem_gc_i=0,
+                               getarrayitem_gc_r=0, getfield_gc_r=0)
+        self.check_resops(getfield_gc_r=1, getarrayitem_gc_i=4,
+                          getfield_gc_i=1)
 
     def test_subclass_of_virtualizable(self):
         myjitdriver = JitDriver(greens = [], reds = ['frame'],
@@ -560,7 +568,8 @@
 
         res = self.meta_interp(f, [10])
         assert res == 55
-        self.check_simple_loop(setfield_gc=0, getfield_gc=0)
+        self.check_simple_loop(setfield_gc=0, getfield_gc_i=0,
+                               getfield_gc_r=0)
 
     def test_external_pass(self):
         jitdriver = JitDriver(greens = [], reds = ['n', 'z', 'frame'],
@@ -1093,7 +1102,8 @@
 
         res = self.meta_interp(f, [10])
         assert res == 55
-        self.check_simple_loop(setfield_gc=0, getfield_gc=0)
+        self.check_simple_loop(setfield_gc=0, getfield_gc_i=0,
+                               getfield_gc_r=0)
 
         from rpython.jit.backend.test.support import BaseCompiledMixin
         if isinstance(self, BaseCompiledMixin):
@@ -1253,8 +1263,10 @@
 
         res = self.meta_interp(f, [10])
         assert res == 155
-        self.check_simple_loop(setfield_gc=0, getfield_gc=0)
-        self.check_resops(setfield_gc=0, getfield_gc=2)
+        self.check_simple_loop(setfield_gc=0, getfield_gc_i=0,
+                               getfield_gc_r=0)
+        self.check_resops(setfield_gc=0, getfield_gc_i=2,
+                          getfield_gc_r=0)
 
     def test_blackhole_should_synchronize(self):
         myjitdriver = JitDriver(greens = [], reds = ['frame'],
@@ -1290,8 +1302,9 @@
 
         res = self.meta_interp(f, [10])
         assert res == 155
-        self.check_simple_loop(setfield_gc=0, getfield_gc=0)
-        self.check_resops(setfield_gc=0, getfield_gc=2)
+        self.check_simple_loop(setfield_gc=0, getfield_gc_i=0,
+                               getfield_gc_r=0)
+        self.check_resops(setfield_gc=0, getfield_gc_i=2, getfield_gc_r=0)
 
     def test_blackhole_should_not_reenter(self):
         if not self.basic:
@@ -1502,7 +1515,7 @@
         res = self.meta_interp(main, [10])
         assert res == main(10)
         self.check_resops({
-            "getfield_gc": 1, "int_lt": 2, "ptr_eq": 1, "guard_true": 3,
+            "getfield_gc_i": 1, "int_lt": 2, "ptr_eq": 1, "guard_true": 3,
             "int_add": 2, "jump": 1
         })
 
@@ -1608,7 +1621,7 @@
         if bridge is not None:
             l = [op for op in
                  bridge.operations if op.getopnum() == rop.SETFIELD_GC]
-            assert "'inst_x'" in str(l[0].getdescr().realdescrref())
+            assert "'inst_x'" in str(l[1].getdescr().realdescrref())
             assert len(l) == 2 # vable token set to null
             l = [op for op in bridge.operations if
                  op.getopnum() == rop.GUARD_NOT_FORCED_2]

From noreply at buildbot.pypy.org  Fri Aug 28 12:03:49 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 12:03:49 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: ups, fix the base class
Message-ID: <20150828100349.029551C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79273:372726fffeb6
Date: 2015-08-28 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/372726fffeb6/

Log:	ups, fix the base class

diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -147,7 +147,7 @@
 
     def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd):
         raise NotImplementedError
-    def compile_and_attach(self, metainterp, new_loop):
+    def compile_and_attach(self, metainterp, new_loop, orig_inputargs):
         raise NotImplementedError
 
 class BasicFinalDescr(AbstractFailDescr):
diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py
--- a/rpython/jit/metainterp/test/test_ajit.py
+++ b/rpython/jit/metainterp/test/test_ajit.py
@@ -2848,7 +2848,7 @@
         # bridge back to the preamble of the first loop is produced. A guard in
         # this bridge is later traced resulting in a failed attempt of retracing
         # the second loop.
-        self.check_trace_count(9)
+        self.check_trace_count(8)
 
         # FIXME: Add a gloabl retrace counter and test that we are not trying more than 5 times.
 

From noreply at buildbot.pypy.org  Fri Aug 28 12:07:45 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 12:07:45 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix the case of bound <-> no
	bound comparisons
Message-ID: <20150828100745.DA3171C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79274:96a9549fcabf
Date: 2015-08-28 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/96a9549fcabf/

Log:	fix the case of bound <-> no bound comparisons

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -3,7 +3,7 @@
 from rpython.jit.metainterp.optimizeopt.info import ArrayPtrInfo,\
      ArrayStructInfo, AbstractStructPtrInfo
 from rpython.jit.metainterp.optimizeopt.intutils import \
-     MININT, MAXINT, IntBound
+     MININT, MAXINT, IntBound, IntLowerBound
 from rpython.jit.metainterp.resoperation import rop, ResOperation,\
      InputArgInt, InputArgRef, InputArgFloat
 from rpython.rlib.debug import debug_print
@@ -391,9 +391,13 @@
 
         extra_guards = state.extra_guards
         cpu = state.cpu
-        if (self.lenbound and other.lenbound and
-                not self.lenbound.contains_bound(other.lenbound)):
-            raise VirtualStatesCantMatch("length bound does not match")
+        if self.lenbound:
+            if other.lenbound is None:
+                other_bound = IntLowerBound(0)
+            else:
+                other_bound = other.lenbound
+            if not self.lenbound.contains_bound(other_bound):
+                raise VirtualStatesCantMatch("length bound does not match")
 
         if self.level == LEVEL_UNKNOWN:
             # confusingly enough, this is done also for pointers

From noreply at buildbot.pypy.org  Fri Aug 28 14:10:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 14:10:32 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix a case of heapcache - now
 breaks some tests because it produces less good code, but seems to work,
 fixing
Message-ID: <20150828121032.745711C0400@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79275:cfa79aa0c03f
Date: 2015-08-28 14:10 +0200
http://bitbucket.org/pypy/pypy/changeset/cfa79aa0c03f/

Log:	fix a case of heapcache - now breaks some tests because it produces
	less good code, but seems to work, fixing

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -123,7 +123,7 @@
                 return constptr
             #
             op.set_forwarded(None)
-            optforce._emit_operation(op)
+            optforce.emit_operation(op)
             newop = optforce.getlastop()
             op.set_forwarded(newop)
             newop.set_forwarded(self)
@@ -187,10 +187,7 @@
                 setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox],
                                           descr=flddescr)
                 self._fields[i] = None
-                if optforce.optheap is not None:
-                    optforce.optheap.propagate_forward(setfieldop)
-                else:
-                    optforce.emit_operation(setfieldop)
+                optforce.emit_operation(setfieldop)
 
     def _force_at_the_end_of_preamble(self, op, optforce, rec):
         if self._fields is None:
@@ -456,10 +453,7 @@
                                  [op, ConstInt(i), subbox],
                                   descr=arraydescr)
             self._items[i] = None
-            if optforce.optheap is not None:
-                optforce.optheap.propagate_forward(setop)
-            else:
-                optforce.emit_operation(setop)
+            optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
     def setitem(self, descr, index, struct, op, cf=None, optheap=None):
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -146,6 +146,9 @@
     def getlastop(self):
         return self.optimizer.getlastop()
 
+    def force_box(self, box):
+        return self.optimizer.force_box(box)
+
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         return self.optimizer.replace_op_with(op, newopnum, args, descr)
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -827,8 +827,8 @@
         escape_n(i3)
         p4 = new_with_vtable(descr=nodesize)
         p1sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, i1, descr=valuedescr)
-        setfield_gc(p1sub, i1, descr=valuedescr)
         setfield_gc(p4, p1sub, descr=nextdescr)
         jump(i1, p4)
         """
@@ -3971,9 +3971,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p2, p3, descr=virtualtokendescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p0, p2, descr=nextdescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p2, p3, descr=virtualtokendescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
@@ -3982,9 +3982,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, 252, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -265,7 +265,8 @@
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:
-            s.enum(virtual_state)
+            if s:
+                s.enum(virtual_state)
 
     def debug_header(self, indent):
         debug_print(indent + 'VArrayStateInfo(%d):' % self.position)
@@ -573,6 +574,8 @@
 
     def make_inputargs(self, inputargs, optimizer, force_boxes=False,
                        append_virtuals=False):
+        if optimizer.optearlyforce:
+            optimizer = optimizer.optearlyforce
         assert len(inputargs) == len(self.state)
         boxes = [None] * self.numnotvirtuals
 

From noreply at buildbot.pypy.org  Fri Aug 28 14:51:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 14:51:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix rpython
Message-ID: <20150828125140.63B8D1C01F4@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79276:74cda9b10634
Date: 2015-08-28 14:51 +0200
http://bitbucket.org/pypy/pypy/changeset/74cda9b10634/

Log:	fix rpython

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -189,9 +189,9 @@
         if self.optimizer.optpure:
             self.optimizer.optpure.pure(opnum, result)
 
-    def pure_from_args(self, opnum, args, result, descr=None):
+    def pure_from_args(self, opnum, args, op, descr=None):
         if self.optimizer.optpure:
-            self.optimizer.optpure.pure_from_args(opnum, args, result, descr)
+            self.optimizer.optpure.pure_from_args(opnum, args, op, descr)
 
     def has_pure_result(self, opnum, args, descr):
         if self.optimizer.optpure:

From noreply at buildbot.pypy.org  Fri Aug 28 14:59:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 14:59:53 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150828125953.7E1E51C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79277:6407bf490893
Date: 2015-08-28 14:59 +0200
http://bitbucket.org/pypy/pypy/changeset/6407bf490893/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -186,7 +186,8 @@
                 subbox = optforce.force_box(fld)
                 setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox],
                                           descr=flddescr)
-                self._fields[i] = None
+                if not flddescr.is_always_pure():
+                    self._fields[i] = None
                 optforce.emit_operation(setfieldop)
 
     def _force_at_the_end_of_preamble(self, op, optforce, rec):
@@ -452,7 +453,8 @@
             setop = ResOperation(rop.SETARRAYITEM_GC,
                                  [op, ConstInt(i), subbox],
                                   descr=arraydescr)
-            self._items[i] = None
+            if not self.arraydescr.is_always_pure():
+                self._items[i] = None
             optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -574,9 +574,20 @@
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
         """
+        expected = """
+        [i1, p2, p3]
+        i3 = getfield_gc_i(p3, descr=valuedescr)
+        escape_n(i3)
+        p1 = new_with_vtable(descr=nodesize)
+        setfield_gc(p1, i1, descr=valuedescr)
+        p1sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p1sub, i1, descr=valuedescr)
+        setfield_gc(p1, p1sub, descr=nextdescr)
+        jump(i1, p1, p2)
+        """
         # The same as test_p123_simple, but with a virtual containing another
         # virtual.
-        self.optimize_loop(ops, ops)
+        self.optimize_loop(ops, expected)
 
     def test_p123_anti_nested(self):
         ops = """
@@ -585,9 +596,9 @@
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
         p2sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
         p1 = new_with_vtable(descr=nodesize)
-        setfield_gc(p2sub, i1, descr=valuedescr)
         jump(i1, p1, p2)
         """
         # The same as test_p123_simple, but in the end the "old" p2 contains
@@ -3049,9 +3060,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p2, p3, descr=virtualtokendescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p0, p2, descr=nextdescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p2, p3, descr=virtualtokendescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [i1]
@@ -3060,9 +3071,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, 252, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         self.optimize_loop(ops, expected)
@@ -3089,9 +3100,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p2, p3, descr=virtualtokendescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p0, p2, descr=nextdescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p2, p3, descr=virtualtokendescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [p2, i1]
@@ -3100,9 +3111,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         # the point of this test is that 'i1' should show up in the fail_args
diff --git a/rpython/jit/metainterp/optimizeopt/vstring.py b/rpython/jit/metainterp/optimizeopt/vstring.py
--- a/rpython/jit/metainterp/optimizeopt/vstring.py
+++ b/rpython/jit/metainterp/optimizeopt/vstring.py
@@ -93,7 +93,7 @@
                 optforce.get_box_replacement(op).set_forwarded(c_s)
                 return c_s
         self._is_virtual = False
-        lengthbox = self.getstrlen(op, optforce.optstring, self.mode)
+        lengthbox = self.getstrlen(op, optforce.optimizer.optstring, self.mode)
         newop = ResOperation(self.mode.NEWSTR, [lengthbox])
         if not we_are_translated():
             newop.name = 'FORCE'

From noreply at buildbot.pypy.org  Fri Aug 28 15:17:11 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 15:17:11 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: dance around a bit
Message-ID: <20150828131711.B8C951C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79278:9217bd619818
Date: 2015-08-28 15:17 +0200
http://bitbucket.org/pypy/pypy/changeset/9217bd619818/

Log:	dance around a bit

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -560,7 +560,7 @@
                     setfieldop = ResOperation(rop.SETINTERIORFIELD_GC,
                                               [op, ConstInt(index), subbox],
                                               descr=flddescr)
-                    optforce._emit_operation(setfieldop)
+                    optforce.emit_operation(setfieldop)
                     # heapcache does not work for interiorfields
                     # if it does, we would need a fix here
                 i += 1
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -303,7 +303,8 @@
             info = self.getptrinfo(box)
             if info is not None and info.is_virtual():
                 rec = {}
-                return info.force_at_the_end_of_preamble(box, self, rec)
+                return info.force_at_the_end_of_preamble(box,
+                                                self.optearlyforce, rec)
             return box
         return box
 
@@ -490,7 +491,7 @@
             return CONST_0
 
     def propagate_all_forward(self, inputargs, ops, call_pure_results=None,
-                              rename_inputargs=True):
+                              rename_inputargs=True, flush=True):
         if rename_inputargs:
             newargs = []
             for inparg in inputargs:
@@ -513,7 +514,8 @@
         #self.loop.operations = self.get_newoperations()
         #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
-        self.flush()
+        if flush:
+            self.flush()
         if extra_jump:
             self.first_optimization.propagate_forward(ops[-1])
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -102,7 +102,8 @@
     def optimize_preamble(self, start_label, end_label, ops, call_pure_results):
         self._check_no_forwarding([[start_label, end_label], ops])
         info, newops = self.optimizer.propagate_all_forward(
-            start_label.getarglist()[:], ops, call_pure_results, True)
+            start_label.getarglist()[:], ops, call_pure_results, True,
+            flush=False)
         exported_state = self.export_state(start_label, end_label.getarglist(),
                                            info.inputargs)
         # we need to absolutely make sure that we've cleaned up all
@@ -117,7 +118,8 @@
         self.potential_extra_ops = {}
         self.optimizer.init_inparg_dict_from(label_args)
         info, _ = self.optimizer.propagate_all_forward(
-            start_label.getarglist()[:], ops, call_pure_results, False)
+            start_label.getarglist()[:], ops, call_pure_results, False,
+            flush=False)
         label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
         target_token = self.finalize_short_preamble(label_op,
                                                     state.virtual_state)

From noreply at buildbot.pypy.org  Fri Aug 28 16:06:53 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 16:06:53 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: merge default
Message-ID: <20150828140653.B42C01C135C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79279:a25583961987
Date: 2015-08-28 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/a25583961987/

Log:	merge default

diff too long, truncating to 2000 out of 17941 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -352,8 +352,7 @@
 Except when otherwise stated (look for LICENSE files or copyright/license
 information at the beginning of each file) the files in the 'lib-python/2.7'
 directory are all copyrighted by the Python Software Foundation and licensed
-under the Python Software License of which you can find a copy here:
-http://www.python.org/doc/Copyright.html 
+under the terms that you can find here: https://docs.python.org/2/license.html
 
 License for 'pypy/module/unicodedata/'
 ======================================
@@ -435,4 +434,4 @@
 
 The code is based on gperftools. You may see a copy of the License for it at
 
-    https://code.google.com/p/gperftools/source/browse/COPYING
+    https://github.com/gperftools/gperftools/blob/master/COPYING
diff --git a/_pytest/assertion/rewrite.py b/_pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include 
 # endif
 #endif
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -236,6 +236,30 @@
             cdecl = self._typeof(cdecl)
         return self._backend.newp(cdecl, init)
 
+    def new_allocator(self, alloc=None, free=None,
+                      should_clear_after_alloc=True):
+        """Return a new allocator, i.e. a function that behaves like ffi.new()
+        but uses the provided low-level 'alloc' and 'free' functions.
+
+        'alloc' is called with the size as argument.  If it returns NULL, a
+        MemoryError is raised.  'free' is called with the result of 'alloc'
+        as argument.  Both can be either Python function or directly C
+        functions.  If 'free' is None, then no free function is called.
+        If both 'alloc' and 'free' are None, the default is used.
+
+        If 'should_clear_after_alloc' is set to False, then the memory
+        returned by 'alloc' is assumed to be already cleared (or you are
+        fine with garbage); otherwise CFFI will clear it.
+        """
+        compiled_ffi = self._backend.FFI()
+        allocator = compiled_ffi.new_allocator(alloc, free,
+                                               should_clear_after_alloc)
+        def allocate(cdecl, init=None):
+            if isinstance(cdecl, basestring):
+                cdecl = self._typeof(cdecl)
+            return allocator(cdecl, init)
+        return allocate
+
     def cast(self, cdecl, source):
         """Similar to a C cast: returns an instance of the named C
         type initialized with the given 'source'.  The source is
@@ -286,7 +310,7 @@
         """
         return self._backend.from_buffer(self.BCharA, python_buffer)
 
-    def callback(self, cdecl, python_callable=None, error=None):
+    def callback(self, cdecl, python_callable=None, error=None, onerror=None):
         """Return a callback object or a decorator making such a
         callback object.  'cdecl' must name a C function pointer type.
         The callback invokes the specified 'python_callable' (which may
@@ -298,7 +322,8 @@
             if not callable(python_callable):
                 raise TypeError("the 'python_callable' argument "
                                 "is not callable")
-            return self._backend.callback(cdecl, python_callable, error)
+            return self._backend.callback(cdecl, python_callable,
+                                          error, onerror)
         if isinstance(cdecl, basestring):
             cdecl = self._typeof(cdecl, consider_function_as_funcptr=True)
         if python_callable is None:
@@ -327,6 +352,13 @@
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
         """
+        try:
+            gcp = self._backend.gcp
+        except AttributeError:
+            pass
+        else:
+            return gcp(cdata, destructor)
+        #
         with self._lock:
             try:
                 gc_weakrefs = self.gc_weakrefs
@@ -428,6 +460,8 @@
             raise TypeError("ffi.include() expects an argument that is also of"
                             " type cffi.FFI, not %r" % (
                                 type(ffi_to_include).__name__,))
+        if ffi_to_include is self:
+            raise ValueError("self.include(self)")
         with ffi_to_include._lock:
             with self._lock:
                 self._parser.include(ffi_to_include._parser)
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -989,7 +989,8 @@
     def cast(self, BType, source):
         return BType._cast_from(source)
 
-    def callback(self, BType, source, error):
+    def callback(self, BType, source, error, onerror):
+        assert onerror is None   # XXX not implemented
         return BType(source, error)
 
     typeof = type
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -53,6 +53,7 @@
 OP_GLOBAL_VAR      = 33
 OP_DLOPEN_FUNC     = 35
 OP_DLOPEN_CONST    = 37
+OP_GLOBAL_VAR_F    = 39
 
 PRIM_VOID          = 0
 PRIM_BOOL          = 1
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -15,9 +15,11 @@
 except ImportError:
     lock = None
 
-_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE)
-_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$",
-                        re.MULTILINE)
+_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
+                        re.DOTALL | re.MULTILINE)
+_r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
+                        r"\b((?:[^\n\\]|\\.)*?)$",
+                        re.DOTALL | re.MULTILINE)
 _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
 _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
@@ -39,6 +41,7 @@
     macros = {}
     for match in _r_define.finditer(csource):
         macroname, macrovalue = match.groups()
+        macrovalue = macrovalue.replace('\\\n', '').strip()
         macros[macroname] = macrovalue
     csource = _r_define.sub('', csource)
     # Replace "[...]" with "[__dotdotdotarray__]"
@@ -423,13 +426,10 @@
                 raise api.CDefError(
                     "%s: a function with only '(...)' as argument"
                     " is not correct C" % (funcname or 'in expression'))
-        elif (len(params) == 1 and
-            isinstance(params[0].type, pycparser.c_ast.TypeDecl) and
-            isinstance(params[0].type.type, pycparser.c_ast.IdentifierType)
-                and list(params[0].type.type.names) == ['void']):
-            del params[0]
         args = [self._as_func_arg(self._get_type(argdeclnode.type))
                 for argdeclnode in params]
+        if not ellipsis and args == [model.void_type]:
+            args = []
         result = self._get_type(typenode.type)
         return model.RawFunctionType(tuple(args), result, ellipsis)
 
@@ -633,6 +633,8 @@
 
     def include(self, other):
         for name, tp in other._declarations.items():
+            if name.startswith('anonymous $enum_$'):
+                continue   # fix for test_anonymous_enum_include
             kind = name.split(' ', 1)[0]
             if kind in ('struct', 'union', 'enum', 'anonymous'):
                 self._declare(name, tp, included=True)
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -35,9 +35,6 @@
     def is_integer_type(self):
         return False
 
-    def sizeof_enabled(self):
-        return False
-
     def get_cached_btype(self, ffi, finishlist, can_delay=False):
         try:
             BType = ffi._cached_btypes[self]
@@ -80,8 +77,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def sizeof_enabled(self):
-        return True
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -205,9 +201,6 @@
 class FunctionPtrType(BaseFunctionType):
     _base_pattern = '(*&)(%s)'
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         result = self.result.get_cached_btype(ffi, finishlist)
         args = []
@@ -233,9 +226,6 @@
             extra = self._base_pattern
         self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra)
 
-    def sizeof_enabled(self):
-        return True
-
     def build_backend_type(self, ffi, finishlist):
         BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True)
         return global_cache(self, ffi, 'new_pointer_type', BItem)
@@ -276,9 +266,6 @@
         self.c_name_with_marker = (
             self.item.c_name_with_marker.replace('&', brackets))
 
-    def sizeof_enabled(self):
-        return self.item.sizeof_enabled() and self.length is not None
-
     def resolve_length(self, newlength):
         return ArrayType(self.item, newlength)
 
@@ -433,9 +420,6 @@
             from . import ffiplatform
             raise ffiplatform.VerificationMissing(self._get_c_name())
 
-    def sizeof_enabled(self):
-        return self.fldtypes is not None
-
     def build_backend_type(self, ffi, finishlist):
         self.check_not_partial()
         finishlist.append(self)
@@ -464,9 +448,6 @@
         self.baseinttype = baseinttype
         self.build_c_name_with_marker()
 
-    def sizeof_enabled(self):
-        return True     # not strictly true, but external enums are obscure
-
     def force_the_name(self, forcename):
         StructOrUnionOrEnum.force_the_name(self, forcename)
         if self.forcename is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -26,6 +26,7 @@
 #define _CFFI_OP_GLOBAL_VAR     33
 #define _CFFI_OP_DLOPEN_FUNC    35
 #define _CFFI_OP_DLOPEN_CONST   37
+#define _CFFI_OP_GLOBAL_VAR_F   39
 
 #define _CFFI_PRIM_VOID          0
 #define _CFFI_PRIM_BOOL          1
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -4,11 +4,6 @@
 
 VERSION = "0x2601"
 
-try:
-    int_type = (int, long)
-except NameError:    # Python 3
-    int_type = int
-
 
 class GlobalExpr:
     def __init__(self, name, address, type_op, size=0, check_value=0):
@@ -981,10 +976,6 @@
         if not self.target_is_python and tp.is_integer_type():
             type_op = CffiOp(OP_CONSTANT_INT, -1)
         else:
-            if not tp.sizeof_enabled():
-                raise ffiplatform.VerificationError(
-                    "constant '%s' is of type '%s', whose size is not known"
-                    % (name, tp._get_c_name()))
             if self.target_is_python:
                 const_kind = OP_DLOPEN_CONST
             else:
@@ -1069,18 +1060,36 @@
         self._do_collect_type(self._global_type(tp, name))
 
     def _generate_cpy_variable_decl(self, tp, name):
-        pass
+        prnt = self._prnt
+        tp = self._global_type(tp, name)
+        if isinstance(tp, model.ArrayType) and tp.length is None:
+            tp = tp.item
+            ampersand = ''
+        else:
+            ampersand = '&'
+        # This code assumes that casts from "tp *" to "void *" is a
+        # no-op, i.e. a function that returns a "tp *" can be called
+        # as if it returned a "void *".  This should be generally true
+        # on any modern machine.  The only exception to that rule (on
+        # uncommon architectures, and as far as I can tell) might be
+        # if 'tp' were a function type, but that is not possible here.
+        # (If 'tp' is a function _pointer_ type, then casts from "fn_t
+        # **" to "void *" are again no-ops, as far as I can tell.)
+        prnt('static ' + tp.get_c_name('*_cffi_var_%s(void)' % (name,)))
+        prnt('{')
+        prnt('  return %s(%s);' % (ampersand, name))
+        prnt('}')
+        prnt()
 
     def _generate_cpy_variable_ctx(self, tp, name):
         tp = self._global_type(tp, name)
         type_index = self._typesdict[tp]
-        type_op = CffiOp(OP_GLOBAL_VAR, type_index)
-        if tp.sizeof_enabled():
-            size = "sizeof(%s)" % (name,)
+        if self.target_is_python:
+            op = OP_GLOBAL_VAR
         else:
-            size = 0
+            op = OP_GLOBAL_VAR_F
         self._lsts["global"].append(
-            GlobalExpr(name, '&%s' % name, type_op, size))
+            GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index)))
 
     # ----------
     # emitting the opcodes for individual types
diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py
--- a/lib_pypy/cffi/setuptools_ext.py
+++ b/lib_pypy/cffi/setuptools_ext.py
@@ -81,10 +81,16 @@
     allsources.extend(kwds.pop('sources', []))
     ext = Extension(name=module_name, sources=allsources, **kwds)
 
-    def make_mod(tmpdir):
+    def make_mod(tmpdir, pre_run=None):
         c_file = os.path.join(tmpdir, module_name + source_extension)
         log.info("generating cffi module %r" % c_file)
         mkpath(tmpdir)
+        # a setuptools-only, API-only hook: called with the "ext" and "ffi"
+        # arguments just before we turn the ffi into C code.  To use it,
+        # subclass the 'distutils.command.build_ext.build_ext' class and
+        # add a method 'def pre_run(self, ext, ffi)'.
+        if pre_run is not None:
+            pre_run(ext, ffi)
         updated = recompiler.make_c_source(ffi, module_name, source, c_file)
         if not updated:
             log.info("already up-to-date")
@@ -98,7 +104,8 @@
     class build_ext_make_mod(base_class):
         def run(self):
             if ext.sources[0] == '$PLACEHOLDER':
-                ext.sources[0] = make_mod(self.build_temp)
+                pre_run = getattr(self, 'pre_run', None)
+                ext.sources[0] = make_mod(self.build_temp, pre_run)
             base_class.run(self)
     dist.cmdclass['build_ext'] = build_ext_make_mod
     # NB. multiple runs here will create multiple 'build_ext_make_mod'
diff --git a/lib_pypy/ctypes_support.py b/lib_pypy/ctypes_support.py
--- a/lib_pypy/ctypes_support.py
+++ b/lib_pypy/ctypes_support.py
@@ -28,7 +28,7 @@
     def _where_is_errno():
         return standard_c_lib.__errno_location()
 
-elif sys.platform in ('darwin', 'freebsd7', 'freebsd8', 'freebsd9'):
+elif sys.platform == 'darwin' or sys.platform.startswith('freebsd'):
     standard_c_lib.__error.restype = ctypes.POINTER(ctypes.c_int)
     standard_c_lib.__error.argtypes = None
     def _where_is_errno():
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -23,14 +23,14 @@
 default_modules.update([
     "_codecs", "gc", "_weakref", "marshal", "errno", "imp", "math", "cmath",
     "_sre", "_pickle_support", "operator", "parser", "symbol", "token", "_ast",
-    "_io", "_random", "__pypy__", "_testing"
+    "_io", "_random", "__pypy__", "_testing", "time"
 ])
 
 
 # --allworkingmodules
 working_modules = default_modules.copy()
 working_modules.update([
-    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd", "time" ,
+    "_socket", "unicodedata", "mmap", "fcntl", "_locale", "pwd",
     "select", "zipimport", "_lsprof", "crypt", "signal", "_rawffi", "termios",
     "zlib", "bz2", "struct", "_hashlib", "_md5", "_sha", "_minimal_curses",
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -135,7 +135,7 @@
 Here are some more technical details.  This issue affects the precise
 time at which ``__del__`` methods are called, which
 is not reliable in PyPy (nor Jython nor IronPython).  It also means that
-weak references may stay alive for a bit longer than expected.  This
+**weak references** may stay alive for a bit longer than expected.  This
 makes "weak proxies" (as returned by ``weakref.proxy()``) somewhat less
 useful: they will appear to stay alive for a bit longer in PyPy, and
 suddenly they will really be dead, raising a ``ReferenceError`` on the
@@ -143,6 +143,24 @@
 ``ReferenceError`` at any place that uses them.  (Or, better yet, don't use
 ``weakref.proxy()`` at all; use ``weakref.ref()``.)
 
+Note a detail in the `documentation for weakref callbacks`__:
+
+    If callback is provided and not None, *and the returned weakref
+    object is still alive,* the callback will be called when the object
+    is about to be finalized.
+
+There are cases where, due to CPython's refcount semantics, a weakref
+dies immediately before or after the objects it points to (typically
+with some circular reference).  If it happens to die just after, then
+the callback will be invoked.  In a similar case in PyPy, both the
+object and the weakref will be considered as dead at the same time,
+and the callback will not be invoked.  (Issue `#2030`__)
+
+.. __: https://docs.python.org/2/library/weakref.html
+.. __: https://bitbucket.org/pypy/pypy/issue/2030/
+
+---------------------------------
+
 There are a few extra implications from the difference in the GC.  Most
 notably, if an object has a ``__del__``, the ``__del__`` is never called more
 than once in PyPy; but CPython will call the same ``__del__`` several times
@@ -321,9 +339,8 @@
 Miscellaneous
 -------------
 
-* Hash randomization (``-R``) is ignored in PyPy.  As documented in
-  http://bugs.python.org/issue14621, some of us believe it has no
-  purpose in CPython either.
+* Hash randomization (``-R``) `is ignored in PyPy`_.  In CPython
+  before 3.4 it has `little point`_.
 
 * You can't store non-string keys in type objects.  For example::
 
@@ -338,7 +355,8 @@
   for about 1400 calls.
 
 * since the implementation of dictionary is different, the exact number
-  which ``__hash__`` and ``__eq__`` are called is different. Since CPython
+  of times that ``__hash__`` and ``__eq__`` are called is different. 
+  Since CPython
   does not give any specific guarantees either, don't rely on it.
 
 * assignment to ``__class__`` is limited to the cases where it
@@ -395,3 +413,12 @@
   interactive mode. In a released version, this behaviour is suppressed, but
   setting the environment variable PYPY_IRC_TOPIC will bring it back. Note that
   downstream package providers have been known to totally disable this feature.
+
+* PyPy's readline module was rewritten from scratch: it is not GNU's
+  readline.  It should be mostly compatible, and it adds multiline
+  support (see ``multiline_input()``).  On the other hand,
+  ``parse_and_bind()`` calls are ignored (issue `#2072`_).
+
+.. _`is ignored in PyPy`: http://bugs.python.org/issue14621
+.. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
+.. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst
--- a/pypy/doc/embedding.rst
+++ b/pypy/doc/embedding.rst
@@ -6,15 +6,9 @@
 C. It was developed in collaboration with Roberto De Ioris from the `uwsgi`_
 project. The `PyPy uwsgi plugin`_ is a good example of using the embedding API.
 
-**NOTE**: As of 1st of December, PyPy comes with ``--shared`` by default
-on linux, linux64 and windows. We will make it the default on all platforms
-by the time of the next release.
-
-The first thing that you need is to compile PyPy yourself with the option
-``--shared``. We plan to make ``--shared`` the default in the future. Consult
-the `how to compile PyPy`_ doc for details. This will result in ``libpypy.so``
-or ``pypy.dll`` file or something similar, depending on your platform. Consult
-your platform specification for details.
+**NOTE**: You need a PyPy compiled with the option ``--shared``, i.e.
+with a ``libpypy-c.so`` or ``pypy-c.dll`` file.  This is the default in
+recent versions of PyPy.
 
 The resulting shared library exports very few functions, however they are
 enough to accomplish everything you need, provided you follow a few principles.
@@ -52,7 +46,11 @@
    source. It'll acquire the GIL.
 
    Note: this is meant to be called *only once* or a few times at most.  See
-   the `more complete example`_ below.
+   the `more complete example`_ below.  In PyPy <= 2.6.0, the globals
+   dictionary is *reused* across multiple calls, giving potentially
+   strange results (e.g. objects dying too early).  In PyPy >= 2.6.1,
+   you get a new globals dictionary for every call (but then, all globals
+   dictionaries are all kept alive forever, in ``sys._pypy_execute_source``).
 
 .. function:: int pypy_execute_source_ptr(char* source, void* ptr);
 
@@ -75,10 +73,12 @@
 Note that this API is a lot more minimal than say CPython C API, so at first
 it's obvious to think that you can't do much. However, the trick is to do
 all the logic in Python and expose it via `cffi`_ callbacks. Let's assume
-we're on linux and pypy is installed in ``/opt/pypy`` with the
+we're on linux and pypy is installed in ``/opt/pypy`` (with
+subdirectories like ``lib-python`` and ``lib_pypy``), and with the
 library in ``/opt/pypy/bin/libpypy-c.so``.  (It doesn't need to be
-installed; you can also replace this path with your local checkout.)
-We write a little C program:
+installed; you can also replace these paths with a local extract of the
+installation tarballs, or with your local checkout of pypy.) We write a
+little C program:
 
 .. code-block:: c
 
@@ -92,7 +92,9 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        /* note: in the path /opt/pypy/x, the final x is ignored and
+           replaced with lib-python and lib_pypy. */
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             printf("Error setting pypy home!\n");
             return 1;
@@ -179,7 +181,7 @@
         int res;
 
         rpython_startup_code();
-        res = pypy_setup_home("/opt/pypy/bin/libpypy-c.so", 1);
+        res = pypy_setup_home("/opt/pypy/x", 1);
         if (res) {
             fprintf(stderr, "Error setting pypy home!\n");
             return -1;
@@ -220,9 +222,15 @@
 Finding pypy_home
 -----------------
 
-Function pypy_setup_home takes one parameter - the path to libpypy. There's 
-currently no "clean" way (pkg-config comes to mind) how to find this path. You 
-can try the following (GNU-specific) hack (don't forget to link against *dl*):
+The function pypy_setup_home() takes as first parameter the path to a
+file from which it can deduce the location of the standard library.
+More precisely, it tries to remove final components until it finds
+``lib-python`` and ``lib_pypy``.  There is currently no "clean" way
+(pkg-config comes to mind) to find this path.  You can try the following
+(GNU-specific) hack (don't forget to link against *dl*), which assumes
+that the ``libpypy-c.so`` is inside the standard library directory.
+(This must more-or-less be the case anyway, otherwise the ``pypy``
+program itself would not run.)
 
 .. code-block:: c
 
@@ -236,7 +244,7 @@
 
     // caller should free returned pointer to avoid memleaks
     // returns NULL on error
-    char* guess_pypyhome() {
+    char* guess_pypyhome(void) {
         // glibc-only (dladdr is why we #define _GNU_SOURCE)
         Dl_info info;
         void *_rpython_startup_code = dlsym(0,"rpython_startup_code");
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -70,6 +70,20 @@
 .. _`use virtualenv (as documented here)`: getting-started.html#installing-using-virtualenv
 
 
+Module xyz does not work in the sandboxed PyPy?
+-----------------------------------------------
+
+You cannot import *any* extension module in a `sandboxed PyPy`_,
+sorry.  Even the built-in modules available are very limited.
+Sandboxing in PyPy is a good proof of concept, really safe IMHO, but
+it is only a proof of concept.  It seriously requires someone working
+on it.  Before this occurs, it can only be used it for "pure Python"
+examples: programs that import mostly nothing (or only pure Python
+modules, recursively).
+
+.. _`sandboxed PyPy`: sandbox.html
+
+
 .. _`See below.`:
 
 Do CPython Extension modules work with PyPy?
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -31,15 +31,14 @@
   and add the new file to  pypy/doc/index-of-whatsnew.rst
 * go to pypy/tool/release and run
   ``force-builds.py ``
-  The following binaries should be built, however, we need more buildbots
- - JIT: windows, linux, os/x, armhf, armel
- - no JIT: windows, linux, os/x
- - sandbox: linux, os/x
+  The following JIT binaries should be built, however, we need more buildbots
+  windows, linux-32, linux-64, osx64, armhf-raring, armhf-raspberrian, armel,
+  freebsd64 
 
 * wait for builds to complete, make sure there are no failures
 * download the builds, repackage binaries. Tag the release version
   and download and repackage source from bitbucket. You may find it
-  convenient to use the ``repackage.sh`` script in pypy/tools to do this. 
+  convenient to use the ``repackage.sh`` script in pypy/tool/release to do this. 
 
   Otherwise repackage and upload source "-src.tar.bz2" to bitbucket
   and to cobra, as some packagers prefer a clearly labeled source package
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-2.6.1.rst
    whatsnew-2.6.0.rst
    whatsnew-2.5.1.rst
    whatsnew-2.5.0.rst
diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -103,12 +103,15 @@
 Howto
 -----
 
-In pypy/goal::
+Grab a copy of the pypy repository_.  In the directory pypy/goal, run::
 
    ../../rpython/bin/rpython -O2 --sandbox targetpypystandalone.py
 
 If you don't have a regular PyPy installed, you should, because it's
-faster to translate, but you can also run ``python translate.py`` instead.
+faster to translate; but you can also run the same line with ``python``
+in front.
+
+.. _repository: https://bitbucket.org/pypy/pypy
 
 
 To run it, use the tools in the pypy/sandbox directory::
@@ -136,8 +139,6 @@
 Not all operations are supported; e.g. if you type os.readlink('...'),
 the controller crashes with an exception and the subprocess is killed.
 Other operations make the subprocess die directly with a "Fatal RPython
-error".  None of this is a security hole; it just means that if you try
-to run some random program, it risks getting killed depending on the
-Python built-in functions it tries to call.  This is a matter of the
-sandboxing layer being incomplete so far, but it should not really be
-a problem in practice.
+error".  None of this is a security hole.  More importantly, *most other
+built-in modules are not enabled.  Please read all the warnings in this
+page before complaining about this.  Contributions welcome.*
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -0,0 +1,76 @@
+========================
+What's new in PyPy 2.6.1
+========================
+
+.. this is a revision shortly after release-2.6.0
+.. startrev: 91904d5c5188
+
+.. branch: use_min_scalar
+Correctly resolve the output dtype of ufunc(array, scalar) calls.
+
+.. branch: stdlib-2.7.10
+
+Update stdlib to version 2.7.10
+
+.. branch: issue2062
+
+.. branch: disable-unroll-for-short-loops
+The JIT no longer performs loop unrolling if the loop compiles to too much code.
+
+.. branch: run-create_cffi_imports
+
+Build cffi import libraries as part of translation by monkey-patching an 
+additional task into translation
+
+.. branch: int-float-list-strategy
+
+Use a compact strategy for Python lists that mix integers and floats,
+at least if the integers fit inside 32 bits.  These lists are now
+stored as an array of floats, like lists that contain only floats; the
+difference is that integers are stored as tagged NaNs.  (This should
+have no visible effect!  After ``lst = [42, 42.5]``, the value of
+``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
+
+.. branch: cffi-callback-onerror
+Part of cffi 1.2.
+
+.. branch: cffi-new-allocator
+Part of cffi 1.2.
+
+.. branch: unicode-dtype
+
+Partial implementation of unicode dtype and unicode scalars.
+
+.. branch: dtypes-compatability
+
+Improve compatibility with numpy dtypes; handle offsets to create unions,
+fix str() and repr(), allow specifying itemsize, metadata and titles, add flags,
+allow subclassing dtype
+
+.. branch: indexing
+
+Refactor array indexing to support ellipses.
+
+.. branch: numpy-docstrings
+
+Allow the docstrings of built-in numpy objects to be set at run-time.
+
+.. branch: nditer-revisited
+
+Implement nditer 'buffered' flag and fix some edge cases
+
+.. branch: ufunc-reduce
+
+Allow multiple axes in ufunc.reduce()
+
+.. branch: fix-tinylang-goals
+
+Update tinylang goals to match current rpython
+
+.. branch: vmprof-review
+
+Clean up of vmprof, notably to handle correctly multiple threads
+
+.. branch: no_boehm_dl
+
+Remove extra link library from Boehm GC
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,12 +2,6 @@
 What's new in PyPy 2.6+
 =======================
 
-.. this is a revision shortly after release-2.6.0
-.. startrev: 91904d5c5188
+.. this is a revision shortly after release-2.6.1
+.. startrev: 07769be4057b
 
-.. branch: use_min_scalar
-Correctly resolve the output dtype of ufunc(array, scalar) calls.
-
-.. branch: stdlib-2.7.10
-
-Update stdlib to version 2.7.10
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -1,6 +1,6 @@
 import py
 
-import os, sys
+import os, sys, subprocess
 
 import pypy
 from pypy.interpreter import gateway
@@ -97,13 +97,16 @@
         from pypy.module.sys.initpath import pypy_find_stdlib
         verbose = rffi.cast(lltype.Signed, verbose)
         if ll_home:
-            home = rffi.charp2str(ll_home)
+            home1 = rffi.charp2str(ll_home)
+            home = os.path.join(home1, 'x') # <- so that 'll_home' can be
+                                            # directly the root directory
         else:
-            home = pypydir
+            home = home1 = pypydir
         w_path = pypy_find_stdlib(space, home)
         if space.is_none(w_path):
             if verbose:
-                debug("Failed to find library based on pypy_find_stdlib")
+                debug("pypy_setup_home: directories 'lib-python' and 'lib_pypy'"
+                      " not found in '%s' or in any parent directory" % home1)
             return rffi.cast(rffi.INT, 1)
         space.startup()
         space.call_function(w_pathsetter, w_path)
@@ -125,13 +128,7 @@
 
     @entrypoint('main', [rffi.CCHARP], c_name='pypy_execute_source')
     def pypy_execute_source(ll_source):
-        after = rffi.aroundstate.after
-        if after: after()
-        source = rffi.charp2str(ll_source)
-        res = _pypy_execute_source(source)
-        before = rffi.aroundstate.before
-        if before: before()
-        return rffi.cast(rffi.INT, res)
+        return pypy_execute_source_ptr(ll_source, 0)
 
     @entrypoint('main', [rffi.CCHARP, lltype.Signed],
                 c_name='pypy_execute_source_ptr')
@@ -139,9 +136,7 @@
         after = rffi.aroundstate.after
         if after: after()
         source = rffi.charp2str(ll_source)
-        space.setitem(w_globals, space.wrap('c_argument'),
-                      space.wrap(ll_ptr))
-        res = _pypy_execute_source(source)
+        res = _pypy_execute_source(source, ll_ptr)
         before = rffi.aroundstate.before
         if before: before()
         return rffi.cast(rffi.INT, res)
@@ -166,15 +161,21 @@
         before = rffi.aroundstate.before
         if before: before()
 
-    w_globals = space.newdict()
-    space.setitem(w_globals, space.wrap('__builtins__'),
-                  space.builtin_modules['__builtin__'])
-
-    def _pypy_execute_source(source):
+    def _pypy_execute_source(source, c_argument):
         try:
-            compiler = space.createcompiler()
-            stmt = compiler.compile(source, 'c callback', 'exec', 0)
-            stmt.exec_code(space, w_globals, w_globals)
+            w_globals = space.newdict(module=True)
+            space.setitem(w_globals, space.wrap('__builtins__'),
+                          space.builtin_modules['__builtin__'])
+            space.setitem(w_globals, space.wrap('c_argument'),
+                          space.wrap(c_argument))
+            space.appexec([space.wrap(source), w_globals], """(src, glob):
+                import sys
+                stmt = compile(src, 'c callback', 'exec')
+                if not hasattr(sys, '_pypy_execute_source'):
+                    sys._pypy_execute_source = []
+                sys._pypy_execute_source.append(glob)
+                exec stmt in glob
+            """)
         except OperationError, e:
             debug("OperationError:")
             debug(" operror-type: " + e.w_type.getname(space))
@@ -294,8 +295,49 @@
         options = make_dict(config)
         wrapstr = 'space.wrap(%r)' % (options)
         pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr
+        if config.objspace.usemodules._cffi_backend:
+            self.hack_for_cffi_modules(driver)
 
         return self.get_entry_point(config)
+    
+    def hack_for_cffi_modules(self, driver):
+        # HACKHACKHACK
+        # ugly hack to modify target goal from compile_c to build_cffi_imports
+        # this should probably get cleaned up and merged with driver.create_exe
+        from rpython.translator.driver import taskdef
+        import types
+
+        class Options(object):
+            pass
+
+
+        def mkexename(name):
+            if sys.platform == 'win32':
+                name = name.new(ext='exe')
+            return name
+
+        @taskdef(['compile_c'], "Create cffi bindings for modules")
+        def task_build_cffi_imports(self):
+            from pypy.tool.build_cffi_imports import create_cffi_import_libraries
+            ''' Use cffi to compile cffi interfaces to modules'''
+            exename = mkexename(driver.compute_exe_name())
+            basedir = exename
+            while not basedir.join('include').exists():
+                _basedir = basedir.dirpath()
+                if _basedir == basedir:
+                    raise ValueError('interpreter %s not inside pypy repo', 
+                                     str(exename))
+                basedir = _basedir
+            modules = self.config.objspace.usemodules.getpaths()
+            options = Options()
+            # XXX possibly adapt options using modules
+            failures = create_cffi_import_libraries(exename, options, basedir)
+            # if failures, they were already printed
+            print  >> sys.stderr, str(exename),'successfully built, but errors while building the above modules will be ignored'
+        driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver)
+        driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, ['compile_c']
+        driver.default_goal = 'build_cffi_imports'
+        # HACKHACKHACK end
 
     def jitpolicy(self, driver):
         from pypy.module.pypyjit.policy import PyPyJitPolicy
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -40,6 +40,11 @@
 PYPYLOG: If set to a non-empty value, enable logging.
 """
 
+try:
+    from __pypy__ import get_hidden_tb, hidden_applevel
+except ImportError:
+    get_hidden_tb = lambda: sys.exc_info()[2]
+    hidden_applevel = lambda f: f
 import sys
 
 DEBUG = False       # dump exceptions before calling the except hook
@@ -63,6 +68,7 @@
             exitcode = 1
     raise SystemExit(exitcode)
 
+ at hidden_applevel
 def run_toplevel(f, *fargs, **fkwds):
     """Calls f() and handles all OperationErrors.
     Intended use is to run the main program or one interactive statement.
@@ -87,13 +93,13 @@
 
     except SystemExit as e:
         handle_sys_exit(e)
-    except:
-        display_exception()
+    except BaseException as e:
+        display_exception(e)
         return False
     return True   # success
 
-def display_exception():
-    etype, evalue, etraceback = sys.exc_info()
+def display_exception(e):
+    etype, evalue, etraceback = type(e), e, get_hidden_tb()
     try:
         # extra debugging info in case the code below goes very wrong
         if DEBUG and hasattr(sys, 'stderr'):
@@ -119,11 +125,11 @@
         hook(etype, evalue, etraceback)
         return # done
 
-    except:
+    except BaseException as e:
         try:
             stderr = sys.stderr
             print >> stderr, 'Error calling sys.excepthook:'
-            originalexcepthook(*sys.exc_info())
+            originalexcepthook(type(e), e, e.__traceback__)
             print >> stderr
             print >> stderr, 'Original exception was:'
         except:
@@ -509,6 +515,7 @@
 
     return options
 
+ at hidden_applevel
 def run_command_line(interactive,
                      inspect,
                      run_command,
@@ -597,6 +604,7 @@
             # Put '' on sys.path
             sys.path.insert(0, '')
 
+            @hidden_applevel
             def run_it():
                 exec run_command in mainmodule.__dict__
             success = run_toplevel(run_it)
@@ -634,6 +642,7 @@
                         print >> sys.stderr, "Could not open PYTHONSTARTUP"
                         print >> sys.stderr, "IOError:", e
                     else:
+                        @hidden_applevel
                         def run_it():
                             co_python_startup = compile(startup,
                                                         python_startup,
@@ -650,6 +659,7 @@
                 inspect = True
             else:
                 # If not interactive, just read and execute stdin normally.
+                @hidden_applevel
                 def run_it():
                     co_stdin = compile(sys.stdin.read(), '', 'exec',
                                        PyCF_ACCEPT_NULL_BYTES)
@@ -689,7 +699,7 @@
     except SystemExit as e:
         status = e.code
         if inspect_requested():
-            display_exception()
+            display_exception(e)
     else:
         status = not success
 
@@ -743,6 +753,7 @@
     # This is important for py3k
     sys.executable = executable
 
+ at hidden_applevel
 def entry_point(executable, argv):
     # note that before calling setup_bootstrap_path, we are limited because we
     # cannot import stdlib modules. In particular, we cannot use unicode
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -11,7 +11,7 @@
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
 
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
-    UserDelAction, CodeUniqueIds)
+    UserDelAction)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
 from pypy.interpreter.argument import Arguments
 from pypy.interpreter.miscutils import ThreadLocals, make_weak_value_dictionary
@@ -200,7 +200,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.buffer_w(space, flags)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def readbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -208,7 +208,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.readbuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def writebuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -216,7 +216,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.writebuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def charbuf_w(self, space):
         w_impl = space.lookup(self, '__buffer__')
@@ -224,7 +224,7 @@
             w_result = space.get_and_call_function(w_impl, self)
             if space.isinstance_w(w_result, space.w_buffer):
                 return w_result.charbuf_w(space)
-        raise TypeError
+        raise BufferInterfaceNotFound
 
     def str_w(self, space):
         self._typed_unwrap_error(space, "string")
@@ -354,6 +354,9 @@
 class DescrMismatch(Exception):
     pass
 
+class BufferInterfaceNotFound(Exception):
+    pass
+
 def wrappable_class_name(Class):
     try:
         return Class.typedef.name
@@ -388,7 +391,6 @@
         self.actionflag = ActionFlag()    # changed by the signal module
         self.check_signal_action = None   # changed by the signal module
         self.user_del_action = UserDelAction(self)
-        self.code_unique_ids = CodeUniqueIds()
         self._code_of_sys_exc_info = None
 
         # can be overridden to a subclass
@@ -667,16 +669,6 @@
             assert ec is not None
             return ec
 
-    def register_code_callback(self, callback):
-        cui = self.code_unique_ids
-        cui.code_callback = callback
-
-    def register_code_object(self, pycode):
-        cui = self.code_unique_ids
-        if cui.code_callback is None:
-            return
-        cui.code_callback(self, pycode)
-
     def _freeze_(self):
         return True
 
@@ -1403,7 +1395,7 @@
         # New buffer interface, returns a buffer based on flags (PyObject_GetBuffer)
         try:
             return w_obj.buffer_w(self, flags)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "'%T' does not have the buffer interface", w_obj)
 
@@ -1411,7 +1403,7 @@
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
             return w_obj.readbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a readable buffer object")
 
@@ -1419,7 +1411,7 @@
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
             return w_obj.writebuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a writeable buffer object")
 
@@ -1427,7 +1419,7 @@
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
         try:
             return w_obj.charbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             raise oefmt(self.w_TypeError,
                         "expected a character buffer object")
 
@@ -1451,11 +1443,11 @@
                 return self.str(w_obj).readbuf_w(self)
             try:
                 return w_obj.buffer_w(self, 0)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 pass
             try:
                 return w_obj.readbuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_str):
@@ -1464,24 +1456,23 @@
                 return self.str(w_obj).str_w(self)
             try:
                 return w_obj.readbuf_w(self).as_str()
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
             try:
-                try:
-                    return w_obj.buffer_w(self, self.BUF_WRITABLE)
-                except OperationError:
-                    self._getarg_error("read-write buffer", w_obj)
-            except TypeError:
+                return w_obj.buffer_w(self, self.BUF_WRITABLE)
+            except OperationError:
+                self._getarg_error("read-write buffer", w_obj)
+            except BufferInterfaceNotFound:
                 pass
             try:
                 return w_obj.writebuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("read-write buffer", w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
-            except TypeError:
+            except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only character buffer", w_obj)
         else:
             assert False
@@ -1503,13 +1494,13 @@
                 raise
         try:
             buf = w_obj.buffer_w(self, 0)
-        except TypeError:
+        except BufferInterfaceNotFound:
             pass
         else:
             return buf.as_str()
         try:
             buf = w_obj.readbuf_w(self)
-        except TypeError:
+        except BufferInterfaceNotFound:
             self._getarg_error("string or buffer", w_obj)
         else:
             return buf.as_str()
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -252,7 +252,8 @@
                                w_t, w_v, w_tb],
                 """(where, objrepr, extra_line, t, v, tb):
                     import sys, traceback
-                    sys.stderr.write('From %s%s:\\n' % (where, objrepr))
+                    if where or objrepr:
+                        sys.stderr.write('From %s%s:\\n' % (where, objrepr))
                     if extra_line:
                         sys.stderr.write(extra_line)
                     traceback.print_exception(t, v, tb)
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -1,6 +1,7 @@
 import sys
 from pypy.interpreter.error import OperationError, get_cleared_operation_error
 from rpython.rlib.unroll import unrolling_iterable
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 
 TICK_COUNTER_STEP = 100
@@ -214,13 +215,21 @@
             self._trace(frame, 'exception', None, operationerr)
         #operationerr.print_detailed_traceback(self.space)
 
-    def sys_exc_info(self): # attn: the result is not the wrapped sys.exc_info() !!!
+    @specialize.arg(1)
+    def sys_exc_info(self, for_hidden=False):
         """Implements sys.exc_info().
-        Return an OperationError instance or None."""
+        Return an OperationError instance or None.
+
+        Ignores exceptions within hidden frames unless for_hidden=True
+        is specified.
+
+        # NOTE: the result is not the wrapped sys.exc_info() !!!
+
+        """
         frame = self.gettopframe()
         while frame:
             if frame.last_exception is not None:
-                if (not frame.hide() or
+                if ((for_hidden or not frame.hide()) or
                         frame.last_exception is
                             get_cleared_operation_error(self.space)):
                     return frame.last_exception
@@ -581,11 +590,3 @@
         # there is no list of length n: if n is large, then the GC
         # will run several times while walking the list, but it will
         # see lower and lower memory usage, with no lower bound of n.
-
-class CodeUniqueIds(object):
-    def __init__(self):
-        if sys.maxint == 2147483647:
-            self.code_unique_id = 0 # XXX this is wrong, it won't work on 32bit
-        else:
-            self.code_unique_id = 0x7000000000000000
-        self.code_callback = None
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -15,7 +15,10 @@
         self.running = False
 
     def descr__repr__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = ''
+        else:
+            code_name = self.pycode.co_name
         addrstring = self.getaddrstring(space)
         return space.wrap("" %
                           (code_name, addrstring))
@@ -45,6 +48,8 @@
         w_framestate, w_running = args_w
         if space.is_w(w_framestate, space.w_None):
             self.frame = None
+            self.space = space
+            self.pycode = None
         else:
             frame = instantiate(space.FrameClass)   # XXX fish
             frame.descr__setstate__(space, w_framestate)
@@ -62,9 +67,10 @@
 
     def send_ex(self, w_arg, operr=None):
         pycode = self.pycode
-        if jit.we_are_jitted() and should_not_inline(pycode):
-            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
-                                                  operr=operr, pycode=pycode)
+        if pycode is not None:
+            if jit.we_are_jitted() and should_not_inline(pycode):
+                generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                    operr=operr, pycode=pycode)
         return self._send_ex(w_arg, operr)
 
     def _send_ex(self, w_arg, operr):
@@ -158,7 +164,10 @@
         return self.pycode
 
     def descr__name__(self, space):
-        code_name = self.pycode.co_name
+        if self.pycode is None:
+            code_name = ''
+        else:
+            code_name = self.pycode.co_name
         return space.wrap(code_name)
 
     # Results can be either an RPython list of W_Root, or it can be an
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -85,7 +85,7 @@
         self.magic = magic
         self._signature = cpython_code_signature(self)
         self._initialize()
-        space.register_code_object(self)
+        self._init_ready()
 
     def _initialize(self):
         if self.co_cellvars:
@@ -127,14 +127,8 @@
             from pypy.objspace.std.mapdict import init_mapdict_cache
             init_mapdict_cache(self)
 
-        cui = self.space.code_unique_ids
-        self._unique_id = cui.code_unique_id
-        cui.code_unique_id += 4  # so we have two bits that we can mark stuff
-        # with
-
-    def _get_full_name(self):
-        return "py:%s:%d:%s" % (self.co_name, self.co_firstlineno,
-                                self.co_filename)
+    def _init_ready(self):
+        "This is a hook for the vmprof module, which overrides this method."
 
     def _cleanup_(self):
         if (self.magic == cpython_magic and
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -711,11 +711,17 @@
         w_item = self.popvalue()
         if self.space.is_w(w_stream, self.space.w_None):
             w_stream = sys_stdout(self.space)   # grumble grumble special cases
-        print_item_to(self.space, w_item, w_stream)
+        print_item_to(self.space, self._printable_object(w_item), w_stream)
 
     def PRINT_ITEM(self, oparg, next_instr):
         w_item = self.popvalue()
-        print_item(self.space, w_item)
+        print_item(self.space, self._printable_object(w_item))
+
+    def _printable_object(self, w_obj):
+        space = self.space
+        if not space.isinstance_w(w_obj, space.w_unicode):
+            w_obj = space.str(w_obj)
+        return w_obj
 
     def PRINT_NEWLINE_TO(self, oparg, next_instr):
         w_stream = self.popvalue()
@@ -1535,9 +1541,9 @@
            stream.write(" ")
 
         # give to write() an argument which is either a string or a unicode
-        # (and let it deals itself with unicode handling)
-        if not isinstance(x, unicode):
-            x = str(x)
+        # (and let it deals itself with unicode handling).  The check "is
+        # unicode" should not use isinstance() at app-level, because that
+        # could be fooled by strange objects, so it is done at interp-level.
         stream.write(x)
 
         # add a softspace unless we just printed a string which ends in a '\t'
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -60,7 +60,6 @@
 
 
 def check_traceback(space, w_tb, msg):
-    from pypy.interpreter.typedef import PyTraceback
     if w_tb is None or not space.isinstance_w(w_tb, space.gettypeobject(PyTraceback.typedef)):
         raise OperationError(space.w_TypeError, space.wrap(msg))
     return w_tb
diff --git a/pypy/interpreter/test/test_interpreter.py b/pypy/interpreter/test/test_interpreter.py
--- a/pypy/interpreter/test/test_interpreter.py
+++ b/pypy/interpreter/test/test_interpreter.py
@@ -299,6 +299,30 @@
         finally:
             sys.stdout = save
 
+    def test_print_strange_object(self):
+        import sys
+
+        class A(object):
+            def __getattribute__(self, name):
+                print "seeing", name
+            def __str__(self):
+                return 'A!!'
+        save = sys.stdout
+        class Out(object):
+            def __init__(self):
+                self.data = []
+            def write(self, x):
+                self.data.append((type(x), x))
+        sys.stdout = out = Out()
+        try:
+            a = A()
+            assert out.data == []
+            print a
+            assert out.data == [(str, 'A!!'),
+                                (str, '\n')]
+        finally:
+            sys.stdout = save
+
     def test_identity(self):
         def f(x): return x
         assert f(666) == 666
diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py
--- a/pypy/interpreter/test/test_zzpickle_and_slow.py
+++ b/pypy/interpreter/test/test_zzpickle_and_slow.py
@@ -491,6 +491,22 @@
         assert pack.mod is result
 
 
+    def test_pickle_generator_crash(self):
+        import pickle
+
+        def f():
+            yield 0
+
+        x = f()
+        x.next()
+        try:
+            x.next()
+        except StopIteration:
+            y = pickle.loads(pickle.dumps(x))
+        assert 'finished' in y.__name__
+        assert 'finished' in repr(y)
+        assert y.gi_code is None
+
 class AppTestGeneratorCloning:
 
     def setup_class(cls):
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -53,6 +53,33 @@
         last = last + x
     return last
 
+
+class _Cons(object):
+    def __init__(self, prev, iter):
+        self.prev = prev
+        self.iter = iter
+
+    def fetch(self):
+        # recursive, loop-less version of the algorithm: works best for a
+        # fixed number of "collections" in the call to map(func, *collections)
+        prev = self.prev
+        if prev is None:
+            args1 = ()
+            stop = True
+        else:
+            args1, stop = prev.fetch()
+        iter = self.iter
+        if iter is None:
+            val = None
+        else:
+            try:
+                val = next(iter)
+                stop = False
+            except StopIteration:
+                self.iter = None
+                val = None
+        return args1 + (val,), stop
+
 def map(func, *collections):
     """map(function, sequence[, sequence, ...]) -> list
 
@@ -69,45 +96,30 @@
     if num_collections == 1:
         if none_func:
             return list(collections[0])
-        # Special case for the really common case of a single collection,
-        # this can be eliminated if we could unroll that loop that creates
-        # `args` based on whether or not len(collections) was constant
+        # Special case for the really common case of a single collection
         seq = collections[0]
         with _ManagedNewlistHint(operator._length_hint(seq, 0)) as result:
             for item in seq:
                 result.append(func(item))
             return result
 
-    # Gather the iterators (pair of (iter, has_finished)) and guess the
+    # Gather the iterators into _Cons objects and guess the
     # result length (the max of the input lengths)
-    iterators = []
+    c = None
     max_hint = 0
     for seq in collections:
-        iterators.append((iter(seq), False))
+        c = _Cons(c, iter(seq))
         max_hint = max(max_hint, operator._length_hint(seq, 0))
 
     with _ManagedNewlistHint(max_hint) as result:
         while True:
-            cont = False
-            args = []
-            for idx, (iterator, has_finished) in enumerate(iterators):
-                val = None
-                if not has_finished:
-                    try:
-                        val = next(iterator)
-                    except StopIteration:
-                        iterators[idx] = (None, True)
-                    else:
-                        cont = True
-                args.append(val)
-            args = tuple(args)
-            if cont:
-                if none_func:
-                    result.append(args)
-                else:
-                    result.append(func(*args))
+            args, stop = c.fetch()
+            if stop:
+                return result
+            if none_func:
+                result.append(args)
             else:
-                return result
+                result.append(func(*args))
 
 class _ManagedNewlistHint(object):
     """ Context manager returning a newlist_hint upon entry.
diff --git a/pypy/module/__builtin__/test/test_abstractinst.py b/pypy/module/__builtin__/test/test_abstractinst.py
--- a/pypy/module/__builtin__/test/test_abstractinst.py
+++ b/pypy/module/__builtin__/test/test_abstractinst.py
@@ -202,3 +202,17 @@
             __subclass__ = set([int])
         assert issubclass(int, Integer)
         assert issubclass(int, (Integer,))
+
+    def test_dont_call_instancecheck_fast_path(self):
+        called = []
+        
+        class M(type):
+            def __instancecheck__(self, obj):
+                called.append("called")
+
+        class C:
+            __metaclass__ = M
+
+        c = C()
+        assert isinstance(c, C)
+        assert not called
diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py
--- a/pypy/module/__builtin__/test/test_functional.py
+++ b/pypy/module/__builtin__/test/test_functional.py
@@ -57,6 +57,11 @@
         b = []
         assert map(lambda x, y: x, a, b) == a
 
+    def test_map_second_item(self):
+        a = []
+        b = [1, 2, 3, 4, 5]
+        assert map(lambda x, y: y, a, b) == b
+
     def test_map_iterables(self):
         class A(object):
             def __init__(self, n):
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -62,6 +62,7 @@
     }
 
     interpleveldefs = {
+        'attach_gdb'                : 'interp_magic.attach_gdb',
         'internal_repr'             : 'interp_magic.internal_repr',
         'bytebuffer'                : 'bytebuffer.bytebuffer',
         'identity_dict'             : 'interp_identitydict.W_IdentityDict',
@@ -71,6 +72,8 @@
         'debug_print_once'          : 'interp_debug.debug_print_once',
         'debug_flush'               : 'interp_debug.debug_flush',
         'builtinify'                : 'interp_magic.builtinify',
+        'hidden_applevel'           : 'interp_magic.hidden_applevel',
+        'get_hidden_tb'             : 'interp_magic.get_hidden_tb',
         'lookup_special'            : 'interp_magic.lookup_special',
         'do_what_I_mean'            : 'interp_magic.do_what_I_mean',
         'validate_fd'               : 'interp_magic.validate_fd',
@@ -98,8 +101,6 @@
 
     def setup_after_space_initialization(self):
         """NOT_RPYTHON"""
-        if not self.space.config.translating:
-            self.extra_interpdef('interp_pdb', 'interp_magic.interp_pdb')
         if self.space.config.objspace.std.withmethodcachecounter:
             self.extra_interpdef('method_cache_counter',
                                  'interp_magic.method_cache_counter')
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -15,12 +15,10 @@
     return space.wrap('%r' % (w_object,))
 
 
-def interp_pdb(space):
-    """Run an interp-level pdb.
-    This is not available in translated versions of PyPy."""
-    assert not we_are_translated()
-    import pdb
-    pdb.set_trace()
+def attach_gdb(space):
+    """Run an interp-level gdb (or pdb when untranslated)"""
+    from rpython.rlib.debug import attach_gdb
+    attach_gdb()
 
 
 @unwrap_spec(name=str)
@@ -59,6 +57,20 @@
     bltn = BuiltinFunction(func)
     return space.wrap(bltn)
 
+def hidden_applevel(space, w_func):
+    """Decorator that hides a function's frame from app-level"""
+    from pypy.interpreter.function import Function
+    func = space.interp_w(Function, w_func)
+    func.getcode().hidden_applevel = True
+    return w_func
+
+def get_hidden_tb(space):
+    """Return the traceback of the current exception being handled by a
+    frame hidden from applevel.
+    """
+    operr = space.getexecutioncontext().sys_exc_info(for_hidden=True)
+    return space.w_None if operr is None else space.wrap(operr.get_traceback())
+
 @unwrap_spec(meth=str)
 def lookup_special(space, w_obj, meth):
     """Lookup up a special method on an object."""
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -27,6 +27,52 @@
         assert A.a is not A.__dict__['a']
         assert A.b is A.__dict__['b']
 
+    def test_hidden_applevel(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def sneak(): (lambda: 1/0)()
+        try:
+            sneak()
+        except ZeroDivisionError as e:
+            tb = sys.exc_info()[2]
+            assert tb.tb_frame == sys._getframe()
+            assert tb.tb_next.tb_frame.f_code.co_name == ''
+        else:
+            assert False, 'Expected ZeroDivisionError'
+
+    def test_hidden_applevel_frames(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def test_hidden():
+            assert sys._getframe().f_code.co_name != 'test_hidden'
+            def e(): 1/0
+            try: e()
+            except ZeroDivisionError as e:
+                assert sys.exc_info() == (None, None, None)
+            else: assert False
+            return 2
+        assert test_hidden() == 2
+
+    def test_get_hidden_tb(self):
+        import __pypy__
+        import sys
+
+        @__pypy__.hidden_applevel
+        def test_hidden_with_tb():
+            def not_hidden(): 1/0
+            try: not_hidden()
+            except ZeroDivisionError as e:
+                assert sys.exc_info() == (None, None, None)
+                tb = __pypy__.get_hidden_tb()
+                assert tb.tb_frame.f_code.co_name == 'not_hidden'
+                return True
+            else: return False
+        assert test_hidden_with_tb()
+
     def test_lookup_special(self):
         from __pypy__ import lookup_special
         class X(object):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -2,7 +2,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib import rdynload
 
-VERSION = "1.1.2"
+VERSION = "1.2.1"
 
 
 class Module(MixedModule):
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -0,0 +1,86 @@
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+
+class W_Allocator(W_Root):
+    _immutable_ = True
+
+    def __init__(self, ffi, w_alloc, w_free, should_clear_after_alloc):
+        self.ffi = ffi    # may be None
+        self.w_alloc = w_alloc
+        self.w_free = w_free
+        self.should_clear_after_alloc = should_clear_after_alloc
+
+    def allocate(self, space, datasize, ctype, length=-1):
+        from pypy.module._cffi_backend import cdataobj, ctypeptr
+        if self.w_alloc is None:
+            if self.should_clear_after_alloc:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=True)
+            else:
+                ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
+                                    flavor='raw', zero=False)
+            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+        else:
+            w_raw_cdata = space.call_function(self.w_alloc,
+                                              space.wrap(datasize))
+            if not isinstance(w_raw_cdata, cdataobj.W_CData):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata object (got %T)",
+                            w_raw_cdata)
+            if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
+                raise oefmt(space.w_TypeError,
+                            "alloc() must return a cdata pointer, not '%s'",
+                            w_raw_cdata.ctype.name)
+            #
+            ptr = w_raw_cdata.unsafe_escaping_ptr()
+            if not ptr:
+                raise oefmt(space.w_MemoryError, "alloc() returned NULL")
+            #
+            if self.should_clear_after_alloc:
+                rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
+                              rffi.cast(rffi.SIZE_T, datasize))
+            #
+            if self.w_free is None:
+                # use this class which does not have a __del__, but still
+                # keeps alive w_raw_cdata
+                res = cdataobj.W_CDataNewNonStdNoFree(space, ptr, ctype, length)
+            else:
+                res = cdataobj.W_CDataNewNonStdFree(space, ptr, ctype, length)
+                res.w_free = self.w_free
+            res.w_raw_cdata = w_raw_cdata
+            return res
+
+    @unwrap_spec(w_init=WrappedDefault(None))
+    def descr_call(self, space, w_arg, w_init):
+        ffi = self.ffi
+        assert ffi is not None
+        w_ctype = ffi.ffi_type(w_arg, ffi.ACCEPT_STRING | ffi.ACCEPT_CTYPE)
+        return w_ctype.newp(w_init, self)
+
+
+W_Allocator.typedef = TypeDef(
+        'FFIAllocator',
+        __call__ = interp2app(W_Allocator.descr_call),
+        )
+W_Allocator.typedef.acceptable_as_base_class = False
+
+
+def new_allocator(ffi, w_alloc, w_free, should_clear_after_alloc):
+    space = ffi.space
+    if space.is_none(w_alloc):
+        w_alloc = None
+    if space.is_none(w_free):
+        w_free = None
+    if w_alloc is None and w_free is not None:
+        raise oefmt(space.w_TypeError, "cannot pass 'free' without 'alloc'")
+    alloc = W_Allocator(ffi, w_alloc, w_free, bool(should_clear_after_alloc))
+    return space.wrap(alloc)
+
+
+default_allocator = W_Allocator(None, None, None, should_clear_after_alloc=True)
+nonzero_allocator = W_Allocator(None, None, None,should_clear_after_alloc=False)
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -22,8 +22,9 @@
 class W_CDataCallback(W_CData):
     #_immutable_fields_ = ...
     ll_error = lltype.nullptr(rffi.CCHARP.TO)
+    w_onerror = None
 
-    def __init__(self, space, ctype, w_callable, w_error):
+    def __init__(self, space, ctype, w_callable, w_error, w_onerror):
         raw_closure = rffi.cast(rffi.CCHARP, clibffi.closureHeap.alloc())
         W_CData.__init__(self, space, raw_closure, ctype)
         #
@@ -31,6 +32,12 @@
             raise oefmt(space.w_TypeError,
                         "expected a callable object, not %T", w_callable)
         self.w_callable = w_callable
+        if not space.is_none(w_onerror):
+            if not space.is_true(space.callable(w_onerror)):
+                raise oefmt(space.w_TypeError,
+                            "expected a callable object for 'onerror', not %T",
+                            w_onerror)
+            self.w_onerror = w_onerror
         #
         fresult = self.getfunctype().ctitem
         size = fresult.size
@@ -161,6 +168,29 @@
 STDERR = 2
 
 
+ at jit.dont_look_inside
+def _handle_applevel_exception(space, callback, e, ll_res, extra_line):
+    callback.write_error_return_value(ll_res)
+    if callback.w_onerror is None:
+        callback.print_error(e, extra_line)
+    else:
+        try:
+            e.normalize_exception(space)
+            w_t = e.w_type
+            w_v = e.get_w_value(space)
+            w_tb = space.wrap(e.get_traceback())
+            w_res = space.call_function(callback.w_onerror,
+                                        w_t, w_v, w_tb)
+            if not space.is_none(w_res):
+                callback.convert_result(ll_res, w_res)
+        except OperationError, e2:
+            # double exception! print a double-traceback...
+            callback.print_error(e, extra_line)    # original traceback
+            e2.write_unraisable(space, '', with_traceback=True,
+                                extra_line="\nDuring the call to 'onerror', "
+                                           "another exception occurred:\n\n")
+
+
 @jit.jit_callback("CFFI")
 def _invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata):
     """ Callback specification.
@@ -178,7 +208,7 @@
         try:
             os.write(STDERR, "SystemError: invoking a callback "
                              "that was already freed\n")
-        except OSError:
+        except:
             pass
         # In this case, we don't even know how big ll_res is.  Let's assume
         # it is just a 'ffi_arg', and store 0 there.
@@ -195,9 +225,7 @@
             extra_line = "Trying to convert the result back to C:\n"
             callback.convert_result(ll_res, w_res)
         except OperationError, e:
-            # got an app-level exception
-            callback.print_error(e, extra_line)
-            callback.write_error_return_value(ll_res)
+            _handle_applevel_exception(space, callback, e, ll_res, extra_line)
         #
     except Exception, e:
         # oups! last-level attempt to recover.
@@ -205,7 +233,7 @@
             os.write(STDERR, "SystemError: callback raised ")
             os.write(STDERR, str(e))
             os.write(STDERR, "\n")
-        except OSError:
+        except:
             pass
         callback.write_error_return_value(ll_res)
     if must_leave:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -363,16 +363,19 @@
     def _sizeof(self):
         return self.ctype.size
 
+    def with_gc(self, w_destructor):
+        with self as ptr:
+            return W_CDataGCP(self.space, ptr, self.ctype, self, w_destructor)
+
 
 class W_CDataMem(W_CData):
-    """This is the base class used for cdata objects that own and free
-    their memory.  Used directly by the results of cffi.cast('int', x)
-    or other primitive explicitly-casted types.  It is further subclassed
-    by W_CDataNewOwning."""
+    """This is used only by the results of cffi.cast('int', x)
+    or other primitive explicitly-casted types."""
     _attrs_ = []
 
-    def __init__(self, space, size, ctype):
-        cdata = lltype.malloc(rffi.CCHARP.TO, size, flavor='raw', zero=True)

From noreply at buildbot.pypy.org  Fri Aug 28 16:59:24 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 16:59:24 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: test and a fix
Message-ID: <20150828145924.79E581C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79280:c3486c27afbd
Date: 2015-08-28 16:59 +0200
http://bitbucket.org/pypy/pypy/changeset/c3486c27afbd/

Log:	test and a fix

diff --git a/rpython/jit/metainterp/optimizeopt/earlyforce.py b/rpython/jit/metainterp/optimizeopt/earlyforce.py
--- a/rpython/jit/metainterp/optimizeopt/earlyforce.py
+++ b/rpython/jit/metainterp/optimizeopt/earlyforce.py
@@ -26,7 +26,7 @@
             not is_raw_free(op, opnum)):
 
             for arg in op.getarglist():
-                self.optimizer.force_box(arg)
+                self.optimizer.force_box(arg, self)
         self.emit_operation(op)
 
     def setup(self):
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -146,8 +146,8 @@
     def getlastop(self):
         return self.optimizer.getlastop()
 
-    def force_box(self, box):
-        return self.optimizer.force_box(box)
+    def force_box(self, box, optforce=None):
+        return self.optimizer.force_box(box, optforce)
 
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         return self.optimizer.replace_op_with(op, newopnum, args, descr)
@@ -336,8 +336,10 @@
             return op
         return op.get_box_replacement(not_const)
 
-    def force_box(self, op):
+    def force_box(self, op, optforce=None):
         op = self.get_box_replacement(op)
+        if optforce is None:
+            optforce = self
         info = op.get_forwarded()
         if self.optunroll and self.optunroll.potential_extra_ops:
             # XXX hack
@@ -351,7 +353,7 @@
         if info is not None:
             if op.type == 'i' and info.is_constant():
                 return ConstInt(info.getint())
-            return info.force_box(op, self)
+            return info.force_box(op, optforce)
         return op
 
     def is_inputarg(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -579,9 +579,9 @@
         i3 = getfield_gc_i(p3, descr=valuedescr)
         escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
-        setfield_gc(p1, i1, descr=valuedescr)
         p1sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p1sub, i1, descr=valuedescr)
+        setfield_gc(p1, i1, descr=valuedescr)
         setfield_gc(p1, p1sub, descr=nextdescr)
         jump(i1, p1, p2)
         """
@@ -5785,5 +5785,25 @@
         """
         self.optimize_loop(ops, expected)
 
+    def test_force_virtual_write(self):
+        ops = """
+        [i1, i2]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        call_n(123, p1, descr=writeadescr)
+        i3 = getfield_gc_i(p1, descr=bdescr)
+        finish(i3)
+        """
+        expected = """
+        [i1, i2]
+        p1 = new(descr=ssize)
+        setfield_gc(p1, i1, descr=adescr)
+        call_n(123, p1, descr=writeadescr)
+        setfield_gc(p1, i2, descr=bdescr)
+        finish(i2)
+        """
+        self.optimize_loop(ops, expected)
+
 class TestLLtype(BaseTestOptimizeBasic, LLtypeMixin):
     pass

From noreply at buildbot.pypy.org  Fri Aug 28 17:37:25 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 17:37:25 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: Add GUARD_GC_TYPE and
 GUARD_NONNULL_GC_TYPE, step 1: llgraph only
Message-ID: <20150828153725.150301C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79281:947d07bc67fc
Date: 2015-08-28 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/947d07bc67fc/

Log:	Add GUARD_GC_TYPE and GUARD_NONNULL_GC_TYPE, step 1: llgraph only

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -16,6 +16,7 @@
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
+from rpython.rlib.objectmodel import Symbolic
 
 class LLTrace(object):
     has_been_freed = False
@@ -88,6 +89,10 @@
     def get_result_type(self):
         return getkind(self.RESULT)[0]
 
+class TypeIDSymbolic(Symbolic):
+    def __init__(self, STRUCT_OR_ARRAY):
+        self.STRUCT_OR_ARRAY = STRUCT_OR_ARRAY
+
 class SizeDescr(AbstractDescr):
     def __init__(self, S, vtable, runner):
         assert not isinstance(vtable, bool)
@@ -114,6 +119,10 @@
     def is_immutable(self):
         return heaptracker.is_immutable_struct(self.S)
 
+    def get_type_id(self):
+        assert isinstance(self.S, lltype.GcStruct)
+        return TypeIDSymbolic(self.S)     # integer-like symbolic
+
     def __repr__(self):
         return 'SizeDescr(%r)' % (self.S,)
 
@@ -222,6 +231,10 @@
         return intbounds.get_integer_max(
             not _is_signed_kind(self.A.OF), rffi.sizeof(self.A.OF))
 
+    def get_type_id(self):
+        assert isinstance(self.A, lltype.GcArray)
+        return TypeIDSymbolic(self.A)     # integer-like symbolic
+
 
 class InteriorFieldDescr(AbstractDescr):
     def __init__(self, A, fieldname, runner):
@@ -281,6 +294,7 @@
     supports_floats = True
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
+    supports_guard_gc_type = True
     translate_support_code = False
     is_llgraph = True
 
@@ -893,6 +907,16 @@
         self.execute_guard_nonnull(descr, arg)
         self.execute_guard_class(descr, arg, klass)
 
+    def execute_guard_gc_type(self, descr, arg, typeid):
+        assert isinstance(typeid, TypeIDSymbolic)
+        TYPE = arg._obj.container._TYPE
+        if TYPE != typeid.STRUCT_OR_ARRAY:
+            self.fail_guard(descr)
+
+    def execute_guard_nonnull_gc_type(self, descr, arg, typeid):
+        self.execute_guard_nonnull(descr, arg)
+        self.execute_guard_gc_type(descr, arg, typeid)
+
     def execute_guard_no_exception(self, descr):
         if self.last_exception is not None:
             self.fail_guard(descr)
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -15,6 +15,7 @@
     # longlongs are supported by the JIT, but stored as doubles.
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
+    supports_guard_gc_type = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4832,3 +4832,49 @@
                                 assert a[i].a == a[i].b == val
                             else:
                                 assert a[i] == rffi.cast(OF, val)
+
+    def test_passing_guard_gc_type_struct(self):
+        if not self.cpu.supports_guard_gc_type:
+            py.test.skip("guard_gc_type not available")
+        t_box, _, descr = self.alloc_instance(self.T)
+        c_typeid = ConstInt(descr.get_type_id())
+        self.execute_operation(rop.GUARD_GC_TYPE, [t_box, c_typeid], 'void')
+        assert not self.guard_failed
+        self.execute_operation(rop.GUARD_NONNULL_GC_TYPE, [t_box, c_typeid],
+                               'void')
+        assert not self.guard_failed
+
+    def test_passing_guard_gc_type_array(self):
+        if not self.cpu.supports_guard_gc_type:
+            py.test.skip("guard_gc_type not available")
+        a_box, A = self.alloc_array_of(rffi.SHORT, 342)
+        arraydescr = self.cpu.arraydescrof(A)
+        c_typeid = ConstInt(arraydescr.get_type_id())
+        self.execute_operation(rop.GUARD_GC_TYPE, [a_box, c_typeid], 'void')
+        assert not self.guard_failed
+        self.execute_operation(rop.GUARD_NONNULL_GC_TYPE, [a_box, c_typeid],
+                               'void')
+        assert not self.guard_failed
+
+    def test_failing_guard_gc_type(self):
+        t_box, _, tdescr = self.alloc_instance(self.T)
+        u_box, _, udescr = self.alloc_instance(self.U)
+        a_box, A = self.alloc_array_of(rffi.SHORT, 342)
+        adescr = self.cpu.arraydescrof(A)
+        c_ttypeid = ConstInt(tdescr.get_type_id())
+        c_utypeid = ConstInt(udescr.get_type_id())
+        c_atypeid = ConstInt(adescr.get_type_id())
+        null_box = self.null_instance()
+        for opname, args in [(rop.GUARD_GC_TYPE, [t_box, c_utypeid]),
+                             (rop.GUARD_GC_TYPE, [u_box, c_ttypeid]),
+                             (rop.GUARD_GC_TYPE, [a_box, c_utypeid]),
+                             (rop.GUARD_GC_TYPE, [t_box, c_atypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [t_box, c_utypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [u_box, c_ttypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [a_box, c_ttypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [u_box, c_atypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [null_box, c_ttypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [null_box, c_atypeid]),
+                             ]:
+            assert self.execute_operation(opname, args, 'void') == None
+            assert self.guard_failed
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -656,6 +656,8 @@
     'GUARD_NONNULL/1d/n',
     'GUARD_ISNULL/1d/n',
     'GUARD_NONNULL_CLASS/2d/n',
+    'GUARD_GC_TYPE/2d/n',
+    'GUARD_NONNULL_GC_TYPE/2d/n',
     '_GUARD_FOLDABLE_LAST',
     'GUARD_NO_EXCEPTION/0d/n',   # may be called with an exception currently set
     'GUARD_EXCEPTION/1d/r',     # may be called with an exception currently set

From noreply at buildbot.pypy.org  Fri Aug 28 18:14:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 18:14:32 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: Add guard_gc_type in the x86
 backend, sharing as much as I can with guard_class.
Message-ID: <20150828161432.674D11C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79282:b1c4f13859b7
Date: 2015-08-28 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/b1c4f13859b7/

Log:	Add guard_gc_type in the x86 backend, sharing as much as I can with
	guard_class.

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -40,6 +40,7 @@
         else:
             translator = None
         self.gc_ll_descr = get_ll_description(gcdescr, translator, rtyper)
+        self.supports_guard_gc_type = bool(translate_support_code)
         if translator and translator.config.translation.gcremovetypeptr:
             self.vtable_offset = None
         else:
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4857,6 +4857,8 @@
         assert not self.guard_failed
 
     def test_failing_guard_gc_type(self):
+        if not self.cpu.supports_guard_gc_type:
+            py.test.skip("guard_gc_type not available")
         t_box, _, tdescr = self.alloc_instance(self.T)
         u_box, _, udescr = self.alloc_instance(self.U)
         a_box, A = self.alloc_array_of(rffi.SHORT, 342)
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1734,10 +1734,10 @@
             self.mc.CMP(locs[0], locs[1])
         self.implement_guard(guard_token, 'NE')
 
-    def _cmp_guard_class(self, locs):
+    def _cmp_guard_class(self, loc_ptr, loc_classptr):
         offset = self.cpu.vtable_offset
         if offset is not None:
-            self.mc.CMP(mem(locs[0], offset), locs[1])
+            self.mc.CMP(mem(loc_ptr, offset), loc_classptr)
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
@@ -1749,26 +1749,38 @@
             #   - multiply by 4 (on 32-bits only) and use it as an
             #     offset in type_info_group
             #   - add 16/32 bytes, to go past the TYPE_INFO structure
-            loc = locs[1]
-            assert isinstance(loc, ImmedLoc)
-            classptr = loc.value
+            assert isinstance(loc_classptr, ImmedLoc)
+            classptr = loc_classptr.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the half-word in the object header.  Note that
-            # this half-word is at offset 0 on a little-endian machine;
-            # it would be at offset 2 or 4 on a big-endian machine.
+            # from reading the half-word in the object header.
             from rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
             type_info_group = rffi.cast(lltype.Signed, type_info_group)
             expected_typeid = classptr - sizeof_ti - type_info_group
-            if IS_X86_32:
-                expected_typeid >>= 2
-                self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid))
-            elif IS_X86_64:
-                self.mc.CMP32_mi((locs[0].value, 0), expected_typeid)
+            self._cmp_guard_gc_type(loc_ptr, ImmedLoc(expected_typeid))
+
+    def _cmp_guard_gc_type(self, loc_ptr, loc_expected_typeid):
+        # Note that the typeid half-word is at offset 0 on a little-endian
+        # machine; it would be at offset 2 or 4 on a big-endian machine.
+        assert self.cpu.supports_guard_gc_type
+        if IS_X86_32:
+            self.mc.CMP16(mem(loc_ptr, 0), loc_expected_typeid)
+        else:
+            self.mc.CMP32(mem(loc_ptr, 0), loc_expected_typeid)
+
+    def _cmp_guard_class_or_gc_type(self, guard_op, locs):
+        if (  guard_op.getopnum() == rop.GUARD_CLASS or
+              guard_op.getopnum() == rop.GUARD_NONNULL_CLASS):
+            self._cmp_guard_class(locs[0], locs[1])
+        elif (guard_op.getopnum() == rop.GUARD_GC_TYPE or
+              guard_op.getopnum() == rop.GUARD_NONNULL_GC_TYPE):
+            self._cmp_guard_gc_type(locs[0], locs[1])
+        else:
+            assert 0
 
     def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2):
-        self._cmp_guard_class(locs)
+        self._cmp_guard_class_or_gc_type(guard_op, locs)
         self.implement_guard(guard_token, 'NE')
 
     def genop_guard_guard_nonnull_class(self, ign_1, guard_op,
@@ -1777,7 +1789,7 @@
         # Patched below
         self.mc.J_il8(rx86.Conditions['B'], 0)
         jb_location = self.mc.get_relative_pos()
-        self._cmp_guard_class(locs)
+        self._cmp_guard_class_or_gc_type(guard_op, locs)
         # patch the JB above
         offset = self.mc.get_relative_pos() - jb_location
         assert 0 < offset <= 127
@@ -1785,6 +1797,9 @@
         #
         self.implement_guard(guard_token, 'NE')
 
+    genop_guard_guard_gc_type = genop_guard_guard_class
+    genop_guard_guard_nonnull_gc_type = genop_guard_guard_nonnull_class
+
     def implement_guard_recovery(self, guard_opnum, faildescr, failargs,
                                  fail_locs, frame_depth):
         exc = (guard_opnum == rop.GUARD_EXCEPTION or
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -428,6 +428,8 @@
         self.perform_guard(op, [x, y], None)
 
     consider_guard_nonnull_class = consider_guard_class
+    consider_guard_gc_type = consider_guard_class
+    consider_guard_nonnull_gc_type = consider_guard_class
 
     def _consider_binop_part(self, op, symm=False):
         x = op.getarg(0)

From noreply at buildbot.pypy.org  Fri Aug 28 18:14:34 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Fri, 28 Aug 2015 18:14:34 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: possible translation fix
Message-ID: <20150828161434.788E81C01F4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79283:fc4b741f41d4
Date: 2015-08-28 17:17 +0100
http://bitbucket.org/pypy/pypy/changeset/fc4b741f41d4/

Log:	possible translation fix

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1767,7 +1767,8 @@
         if IS_X86_32:
             self.mc.CMP16(mem(loc_ptr, 0), loc_expected_typeid)
         else:
-            self.mc.CMP32(mem(loc_ptr, 0), loc_expected_typeid)
+            assert isinstance(loc_expected_typeid, ImmedLoc)
+            self.mc.CMP32_mi((loc_ptr.value, 0), loc_expected_typeid.value)
 
     def _cmp_guard_class_or_gc_type(self, guard_op, locs):
         if (  guard_op.getopnum() == rop.GUARD_CLASS or

From noreply at buildbot.pypy.org  Fri Aug 28 18:32:32 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 18:32:32 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: kill modifying short preamble
	in place
Message-ID: <20150828163232.DB0341C13F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79284:4cf54bb12f0f
Date: 2015-08-28 18:14 +0200
http://bitbucket.org/pypy/pypy/changeset/4cf54bb12f0f/

Log:	kill modifying short preamble in place

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -408,8 +408,7 @@
 
     def build_short_preamble(self):
         label_op = ResOperation(rop.LABEL, self.short_inputargs[:])
-        jump_op = ResOperation(rop.JUMP, self.short_preamble_jump)
-        # WARNING! the short_preamble_jump is shared on purpose
+        jump_op = ResOperation(rop.JUMP, self.short_preamble_jump[:])
         if not we_are_translated():
             TreeLoop.check_consistency_of(self.short_inputargs,
                                 self.short + [jump_op], check_descr=False)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -855,19 +855,19 @@
         i3 = getfield_gc_i(p3sub, descr=valuedescr)
         escape_n(i3)
         p2sub = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2sub, i1, descr=valuedescr)
         setfield_gc(p2, p2sub, descr=nextdescr)
-        setfield_gc(p2sub, i1, descr=valuedescr)
-        i4 = same_as_i(i1)
-        jump(i1, p2, p2sub, i4)
-        """
-        expected = """
-        [i1, p2, p10, i10]
-        escape_n(i10)
+        jump(i1, p2, p2sub)
+        """
+        expected = """
+        [i1, p2, p10]
+        i3 = getfield_gc_i(p10, descr=valuedescr)
+        escape_n(i3)
         p1 = new_with_vtable(descr=nodesize)
         p3sub = new_with_vtable(descr=nodesize2)
         setfield_gc(p3sub, i1, descr=valuedescr)
         setfield_gc(p1, p3sub, descr=nextdescr)
-        jump(i1, p1, p3sub, i1)
+        jump(i1, p1, p3sub)
         """
         self.optimize_loop(ops, expected, preamble,
                 jump_values=[None, self.nodefulladdr, self.nodefulladdr, None])
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -81,7 +81,6 @@
     distinction anymore)"""
 
     short_preamble_producer = None
-    main_target_token = None
 
     def __init__(self, metainterp_sd, jitdriver_sd, optimizations):
         self.optimizer = UnrollableOptimizer(metainterp_sd, jitdriver_sd,
@@ -121,20 +120,15 @@
             start_label.getarglist()[:], ops, call_pure_results, False,
             flush=False)
         label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
+        extra_same_as = self.short_preamble_producer.extra_same_as[:]
         target_token = self.finalize_short_preamble(label_op,
                                                     state.virtual_state)
-        self.main_target_token = target_token
         label_op.setdescr(target_token)
-        extra = self.short_preamble_producer.used_boxes
-        label_op.initarglist(label_args + extra)
         # force the boxes for virtual state to match
         state.virtual_state.make_inputargs(
             [self.get_box_replacement(x) for x in end_jump.getarglist()],
             self.optimizer, force_boxes=True)
         new_virtual_state = self.jump_to_existing_trace(end_jump)
-        # the short preamble could have grown, we reinitialize
-        # the label_op again
-        label_op.initarglist(label_args + extra)
         if new_virtual_state is not None:
             celltoken = start_label.getdescr()
             assert isinstance(celltoken, JitCellToken)
@@ -142,8 +136,7 @@
             return (UnrollInfo(target_token, label_op, []),
                     self.optimizer._newoperations)
             #return new_virtual_state, self.optimizer._newoperations
-        return (UnrollInfo(target_token, label_op,
-                           self.short_preamble_producer.extra_same_as),
+        return (UnrollInfo(target_token, label_op, extra_same_as),
                 self.optimizer._newoperations)
 
     def optimize_bridge(self, start_label, operations, call_pure_results,
@@ -203,6 +196,8 @@
         target_token.virtual_state = virtual_state
         target_token.short_preamble = short_preamble
         jitcelltoken.target_tokens.append(target_token)
+        self.short_preamble_producer = None # no more boxes
+        label_op.initarglist(label_op.getarglist() + sb.used_boxes)
         return target_token
 
     def jump_to_preamble(self, cell_token, jump_op, info):
@@ -238,13 +233,9 @@
                 self.optimizer, append_virtuals=True)
             args = target_virtual_state.make_inputargs(args,
                 self.optimizer)
-            if target_token is self.main_target_token:
-                # rebuild the short preamble, it might have changed
-                new_sp = self.short_preamble_producer.build_short_preamble()
-                target_token.short_preamble = new_sp
             short_preamble = target_token.short_preamble
             extra = self.inline_short_preamble(pass_to_short, args,
-                short_preamble[0].getarglist(), short_preamble,
+                short_preamble[0].getarglist(), short_preamble[1:-1],
                 short_preamble[-1].getarglist(), self.optimizer.patchguardop)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
@@ -255,16 +246,13 @@
     def inline_short_preamble(self, jump_args, args_no_virtuals,
                               short_inputargs, short_ops,
                               short_jump_op, patchguardop):
-        # warning!!!! short_jump_op might have arguments appended IN PLACE
         try:
             self._check_no_forwarding([short_inputargs, short_ops], False)
             assert len(short_inputargs) == len(jump_args)
             for i in range(len(jump_args)):
                 short_inputargs[i].set_forwarded(None)
                 self.make_equal_to(short_inputargs[i], jump_args[i])
-            i = 1
-            while i < len(short_ops) - 1:
-                op = short_ops[i]
+            for op in short_ops:
                 if op.is_guard():
                     op = self.replace_op_with(op, op.getopnum(),
                                     descr=compile.ResumeAtPositionDescr())
@@ -272,7 +260,6 @@
                     op.rd_snapshot = patchguardop.rd_snapshot
                     op.rd_frame_info_list = patchguardop.rd_frame_info_list
                 self.optimizer.send_extra_operation(op)
-                i += 1
             # force all of them except the virtuals
             for arg in args_no_virtuals + short_jump_op:
                 self.optimizer.force_box(self.get_box_replacement(arg))
@@ -284,9 +271,9 @@
                 op.set_forwarded(None)
 
     def export_state(self, start_label, original_label_args, renamed_inputargs):
-        self.optimizer.force_at_end_of_preamble()
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
+        self.optimizer.force_at_end_of_preamble()
         virtual_state = self.get_virtual_state(end_args)
         end_args = [self.get_box_replacement(arg) for arg in end_args]
         infos = {}

From noreply at buildbot.pypy.org  Fri Aug 28 18:32:35 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 18:32:35 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: work a bit on tests
Message-ID: <20150828163235.017401C13F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79285:e476d8be1785
Date: 2015-08-28 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/e476d8be1785/

Log:	work a bit on tests

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -258,6 +258,8 @@
         self.optearlyforce = None
         self.optunroll = None
 
+        self._emitting = True
+
         self.set_optimizations(optimizations)
         self.setup()
 
@@ -576,8 +578,9 @@
                         self.force_box(farg)
         elif op.can_raise():
             self.exception_might_have_happened = True
-        self._really_emitted_operation = op
-        self._newoperations.append(op)
+        if self._emitting:
+            self._really_emitted_operation = op
+            self._newoperations.append(op)
 
     def getlastop(self):
         return self._really_emitted_operation
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -1225,22 +1225,24 @@
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(descr=nodesize2)
         p2 = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
+        setfield_gc(p1, p2, descr=nextdescr)
         setfield_gc(p0, p1, descr=nextdescr)
-        setfield_gc(p2, i1, descr=valuedescr)
-        setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1, p2, i1)
+        jump(p1)
         """
         loop = """
-        [p0, p41, i0]
+        [p0]
+        p41 = getfield_gc_r(p0, descr=nextdescr)
+        i0 = getfield_gc_i(p41, descr=valuedescr)
         i1 = int_add(i0, 1)
         p1 = new_with_vtable(descr=nodesize2)
         p2 = new_with_vtable(descr=nodesize2)
+        setfield_gc(p2, i1, descr=valuedescr)
         setfield_gc(p2, p1, descr=nextdescr)
         setfield_gc(p0, p1, descr=nextdescr)
-        setfield_gc(p2, i1, descr=valuedescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        jump(p1, p2, i1)
+        jump(p1)
         """
         self.optimize_loop(ops, loop, preamble)
 
@@ -1294,8 +1296,8 @@
         i28 = int_add(i0, 1)
         i29 = int_add(i28, 1)
         p30 = new_with_vtable(descr=nodesize)
+        setfield_gc(p30, i28, descr=valuedescr)
         setfield_gc(p3, p30, descr=nextdescr)
-        setfield_gc(p30, i28, descr=valuedescr)
         #p46 = same_as(p30) # This same_as should be killed by backend
         jump(i29, p30, p3)
         """
@@ -1304,8 +1306,8 @@
         i28 = int_add(i0, 1)
         i29 = int_add(i28, 1)
         p30 = new_with_vtable(descr=nodesize)
+        setfield_gc(p30, i28, descr=valuedescr)
         setfield_gc(p3, p30, descr=nextdescr)
-        setfield_gc(p30, i28, descr=valuedescr)
         jump(i29, p30, p3)
         """
         self.optimize_loop(ops, expected, preamble)
@@ -2719,8 +2721,8 @@
         guard_true(i3) [p1, p4]
         i4 = int_neg(i2)
         p2 = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        setfield_gc(p2, p4, descr=nextdescr)
         #i101 = same_as(i4)
         jump(p1, i2, i4, p4) #, i101)
         """
@@ -2728,8 +2730,8 @@
         [p1, i2, i4, p4] #, i5]
         guard_true(i4) [p1, p4]
         p2 = new_with_vtable(descr=nodesize)
+        setfield_gc(p2, p4, descr=nextdescr)
         setfield_gc(p1, p2, descr=nextdescr)
-        setfield_gc(p2, p4, descr=nextdescr)
         jump(p1, i2, 1, p4) #, i5)
         """
         self.optimize_loop(ops, expected, preamble)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -13,7 +13,7 @@
 from rpython.jit.metainterp import compile
 from rpython.rlib.debug import debug_print
 
-class UnrollableOptimizer(Optimizer):
+class UnrollableOptimizer(Optimizer):    
     def force_op_from_preamble(self, preamble_op):
         if isinstance(preamble_op, PreambleOp):
             op = preamble_op.op

From noreply at buildbot.pypy.org  Fri Aug 28 18:32:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 18:32:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: merge
Message-ID: <20150828163237.204CD1C13F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79286:9e0692ce2a85
Date: 2015-08-28 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/9e0692ce2a85/

Log:	merge

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -16,6 +16,7 @@
 
 from rpython.rlib.clibffi import FFI_DEFAULT_ABI
 from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong
+from rpython.rlib.objectmodel import Symbolic
 
 class LLTrace(object):
     has_been_freed = False
@@ -88,6 +89,10 @@
     def get_result_type(self):
         return getkind(self.RESULT)[0]
 
+class TypeIDSymbolic(Symbolic):
+    def __init__(self, STRUCT_OR_ARRAY):
+        self.STRUCT_OR_ARRAY = STRUCT_OR_ARRAY
+
 class SizeDescr(AbstractDescr):
     def __init__(self, S, vtable, runner):
         assert not isinstance(vtable, bool)
@@ -114,6 +119,10 @@
     def is_immutable(self):
         return heaptracker.is_immutable_struct(self.S)
 
+    def get_type_id(self):
+        assert isinstance(self.S, lltype.GcStruct)
+        return TypeIDSymbolic(self.S)     # integer-like symbolic
+
     def __repr__(self):
         return 'SizeDescr(%r)' % (self.S,)
 
@@ -222,6 +231,10 @@
         return intbounds.get_integer_max(
             not _is_signed_kind(self.A.OF), rffi.sizeof(self.A.OF))
 
+    def get_type_id(self):
+        assert isinstance(self.A, lltype.GcArray)
+        return TypeIDSymbolic(self.A)     # integer-like symbolic
+
 
 class InteriorFieldDescr(AbstractDescr):
     def __init__(self, A, fieldname, runner):
@@ -281,6 +294,7 @@
     supports_floats = True
     supports_longlong = r_uint is not r_ulonglong
     supports_singlefloats = True
+    supports_guard_gc_type = True
     translate_support_code = False
     is_llgraph = True
 
@@ -893,6 +907,16 @@
         self.execute_guard_nonnull(descr, arg)
         self.execute_guard_class(descr, arg, klass)
 
+    def execute_guard_gc_type(self, descr, arg, typeid):
+        assert isinstance(typeid, TypeIDSymbolic)
+        TYPE = arg._obj.container._TYPE
+        if TYPE != typeid.STRUCT_OR_ARRAY:
+            self.fail_guard(descr)
+
+    def execute_guard_nonnull_gc_type(self, descr, arg, typeid):
+        self.execute_guard_nonnull(descr, arg)
+        self.execute_guard_gc_type(descr, arg, typeid)
+
     def execute_guard_no_exception(self, descr):
         if self.last_exception is not None:
             self.fail_guard(descr)
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -40,6 +40,7 @@
         else:
             translator = None
         self.gc_ll_descr = get_ll_description(gcdescr, translator, rtyper)
+        self.supports_guard_gc_type = bool(translate_support_code)
         if translator and translator.config.translation.gcremovetypeptr:
             self.vtable_offset = None
         else:
diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py
--- a/rpython/jit/backend/model.py
+++ b/rpython/jit/backend/model.py
@@ -15,6 +15,7 @@
     # longlongs are supported by the JIT, but stored as doubles.
     # Boxes and Consts are BoxFloats and ConstFloats.
     supports_singlefloats = False
+    supports_guard_gc_type = False
 
     propagate_exception_descr = None
 
diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py
--- a/rpython/jit/backend/test/runner_test.py
+++ b/rpython/jit/backend/test/runner_test.py
@@ -4832,3 +4832,51 @@
                                 assert a[i].a == a[i].b == val
                             else:
                                 assert a[i] == rffi.cast(OF, val)
+
+    def test_passing_guard_gc_type_struct(self):
+        if not self.cpu.supports_guard_gc_type:
+            py.test.skip("guard_gc_type not available")
+        t_box, _, descr = self.alloc_instance(self.T)
+        c_typeid = ConstInt(descr.get_type_id())
+        self.execute_operation(rop.GUARD_GC_TYPE, [t_box, c_typeid], 'void')
+        assert not self.guard_failed
+        self.execute_operation(rop.GUARD_NONNULL_GC_TYPE, [t_box, c_typeid],
+                               'void')
+        assert not self.guard_failed
+
+    def test_passing_guard_gc_type_array(self):
+        if not self.cpu.supports_guard_gc_type:
+            py.test.skip("guard_gc_type not available")
+        a_box, A = self.alloc_array_of(rffi.SHORT, 342)
+        arraydescr = self.cpu.arraydescrof(A)
+        c_typeid = ConstInt(arraydescr.get_type_id())
+        self.execute_operation(rop.GUARD_GC_TYPE, [a_box, c_typeid], 'void')
+        assert not self.guard_failed
+        self.execute_operation(rop.GUARD_NONNULL_GC_TYPE, [a_box, c_typeid],
+                               'void')
+        assert not self.guard_failed
+
+    def test_failing_guard_gc_type(self):
+        if not self.cpu.supports_guard_gc_type:
+            py.test.skip("guard_gc_type not available")
+        t_box, _, tdescr = self.alloc_instance(self.T)
+        u_box, _, udescr = self.alloc_instance(self.U)
+        a_box, A = self.alloc_array_of(rffi.SHORT, 342)
+        adescr = self.cpu.arraydescrof(A)
+        c_ttypeid = ConstInt(tdescr.get_type_id())
+        c_utypeid = ConstInt(udescr.get_type_id())
+        c_atypeid = ConstInt(adescr.get_type_id())
+        null_box = self.null_instance()
+        for opname, args in [(rop.GUARD_GC_TYPE, [t_box, c_utypeid]),
+                             (rop.GUARD_GC_TYPE, [u_box, c_ttypeid]),
+                             (rop.GUARD_GC_TYPE, [a_box, c_utypeid]),
+                             (rop.GUARD_GC_TYPE, [t_box, c_atypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [t_box, c_utypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [u_box, c_ttypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [a_box, c_ttypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [u_box, c_atypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [null_box, c_ttypeid]),
+                             (rop.GUARD_NONNULL_GC_TYPE, [null_box, c_atypeid]),
+                             ]:
+            assert self.execute_operation(opname, args, 'void') == None
+            assert self.guard_failed
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1734,10 +1734,10 @@
             self.mc.CMP(locs[0], locs[1])
         self.implement_guard(guard_token, 'NE')
 
-    def _cmp_guard_class(self, locs):
+    def _cmp_guard_class(self, loc_ptr, loc_classptr):
         offset = self.cpu.vtable_offset
         if offset is not None:
-            self.mc.CMP(mem(locs[0], offset), locs[1])
+            self.mc.CMP(mem(loc_ptr, offset), loc_classptr)
         else:
             # XXX hard-coded assumption: to go from an object to its class
             # we use the following algorithm:
@@ -1749,26 +1749,39 @@
             #   - multiply by 4 (on 32-bits only) and use it as an
             #     offset in type_info_group
             #   - add 16/32 bytes, to go past the TYPE_INFO structure
-            loc = locs[1]
-            assert isinstance(loc, ImmedLoc)
-            classptr = loc.value
+            assert isinstance(loc_classptr, ImmedLoc)
+            classptr = loc_classptr.value
             # here, we have to go back from 'classptr' to the value expected
-            # from reading the half-word in the object header.  Note that
-            # this half-word is at offset 0 on a little-endian machine;
-            # it would be at offset 2 or 4 on a big-endian machine.
+            # from reading the half-word in the object header.
             from rpython.memory.gctypelayout import GCData
             sizeof_ti = rffi.sizeof(GCData.TYPE_INFO)
             type_info_group = llop.gc_get_type_info_group(llmemory.Address)
             type_info_group = rffi.cast(lltype.Signed, type_info_group)
             expected_typeid = classptr - sizeof_ti - type_info_group
-            if IS_X86_32:
-                expected_typeid >>= 2
-                self.mc.CMP16(mem(locs[0], 0), ImmedLoc(expected_typeid))
-            elif IS_X86_64:
-                self.mc.CMP32_mi((locs[0].value, 0), expected_typeid)
+            self._cmp_guard_gc_type(loc_ptr, ImmedLoc(expected_typeid))
+
+    def _cmp_guard_gc_type(self, loc_ptr, loc_expected_typeid):
+        # Note that the typeid half-word is at offset 0 on a little-endian
+        # machine; it would be at offset 2 or 4 on a big-endian machine.
+        assert self.cpu.supports_guard_gc_type
+        if IS_X86_32:
+            self.mc.CMP16(mem(loc_ptr, 0), loc_expected_typeid)
+        else:
+            assert isinstance(loc_expected_typeid, ImmedLoc)
+            self.mc.CMP32_mi((loc_ptr.value, 0), loc_expected_typeid.value)
+
+    def _cmp_guard_class_or_gc_type(self, guard_op, locs):
+        if (  guard_op.getopnum() == rop.GUARD_CLASS or
+              guard_op.getopnum() == rop.GUARD_NONNULL_CLASS):
+            self._cmp_guard_class(locs[0], locs[1])
+        elif (guard_op.getopnum() == rop.GUARD_GC_TYPE or
+              guard_op.getopnum() == rop.GUARD_NONNULL_GC_TYPE):
+            self._cmp_guard_gc_type(locs[0], locs[1])
+        else:
+            assert 0
 
     def genop_guard_guard_class(self, ign_1, guard_op, guard_token, locs, ign_2):
-        self._cmp_guard_class(locs)
+        self._cmp_guard_class_or_gc_type(guard_op, locs)
         self.implement_guard(guard_token, 'NE')
 
     def genop_guard_guard_nonnull_class(self, ign_1, guard_op,
@@ -1777,7 +1790,7 @@
         # Patched below
         self.mc.J_il8(rx86.Conditions['B'], 0)
         jb_location = self.mc.get_relative_pos()
-        self._cmp_guard_class(locs)
+        self._cmp_guard_class_or_gc_type(guard_op, locs)
         # patch the JB above
         offset = self.mc.get_relative_pos() - jb_location
         assert 0 < offset <= 127
@@ -1785,6 +1798,9 @@
         #
         self.implement_guard(guard_token, 'NE')
 
+    genop_guard_guard_gc_type = genop_guard_guard_class
+    genop_guard_guard_nonnull_gc_type = genop_guard_guard_nonnull_class
+
     def implement_guard_recovery(self, guard_opnum, faildescr, failargs,
                                  fail_locs, frame_depth):
         exc = (guard_opnum == rop.GUARD_EXCEPTION or
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -428,6 +428,8 @@
         self.perform_guard(op, [x, y], None)
 
     consider_guard_nonnull_class = consider_guard_class
+    consider_guard_gc_type = consider_guard_class
+    consider_guard_nonnull_gc_type = consider_guard_class
 
     def _consider_binop_part(self, op, symm=False):
         x = op.getarg(0)
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -656,6 +656,8 @@
     'GUARD_NONNULL/1d/n',
     'GUARD_ISNULL/1d/n',
     'GUARD_NONNULL_CLASS/2d/n',
+    'GUARD_GC_TYPE/2d/n',
+    'GUARD_NONNULL_GC_TYPE/2d/n',
     '_GUARD_FOLDABLE_LAST',
     'GUARD_NO_EXCEPTION/0d/n',   # may be called with an exception currently set
     'GUARD_EXCEPTION/1d/r',     # may be called with an exception currently set

From noreply at buildbot.pypy.org  Fri Aug 28 20:04:37 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 20:04:37 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix rpython
Message-ID: <20150828180437.B89371C13F7@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79287:2c6a9ab1afcd
Date: 2015-08-28 20:04 +0200
http://bitbucket.org/pypy/pypy/changeset/2c6a9ab1afcd/

Log:	fix rpython

diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -146,8 +146,8 @@
     def getlastop(self):
         return self.optimizer.getlastop()
 
-    def force_box(self, box, optforce=None):
-        return self.optimizer.force_box(box, optforce)
+    def force_box(self, op, optforce=None):
+        return self.optimizer.force_box(op, optforce)
 
     def replace_op_with(self, op, newopnum, args=None, descr=None):
         return self.optimizer.replace_op_with(op, newopnum, args, descr)

From noreply at buildbot.pypy.org  Fri Aug 28 20:13:20 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Fri, 28 Aug 2015 20:13:20 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: bah
Message-ID: <20150828181320.030C11C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79288:fa5a35ae6ee4
Date: 2015-08-28 20:13 +0200
http://bitbucket.org/pypy/pypy/changeset/fa5a35ae6ee4/

Log:	bah

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -234,8 +234,10 @@
             args = target_virtual_state.make_inputargs(args,
                 self.optimizer)
             short_preamble = target_token.short_preamble
+            lgt = len(short_preamble) - 1
+            assert lgt >= 0
             extra = self.inline_short_preamble(pass_to_short, args,
-                short_preamble[0].getarglist(), short_preamble[1:-1],
+                short_preamble[0].getarglist(), short_preamble[1:lgt],
                 short_preamble[-1].getarglist(), self.optimizer.patchguardop)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,

From noreply at buildbot.pypy.org  Sat Aug 29 20:56:49 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sat, 29 Aug 2015 20:56:49 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: Added tag release-2.6.1 for
	changeset f3ad1e1e1d62
Message-ID: <20150829185649.400241C122B@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79289:c561da905598
Date: 2015-08-29 21:57 +0300
http://bitbucket.org/pypy/pypy/changeset/c561da905598/

Log:	Added tag release-2.6.1 for changeset f3ad1e1e1d62

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -15,3 +15,4 @@
 e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
 e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
 295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
+f3ad1e1e1d6215e20d34bb65ab85ff9188c9f559 release-2.6.1

From noreply at buildbot.pypy.org  Sat Aug 29 22:31:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 29 Aug 2015 22:31:49 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <20150829203149.996911C0FD4@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r630:dc68925aaaaf
Date: 2015-08-29 22:32 +0200
http://bitbucket.org/pypy/pypy.org/changeset/dc68925aaaaf/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 
 
    
-   $60016 of $105000 (57.2%)
+   $60026 of $105000 (57.2%)
    
    
    
    
 
@@ -23,7 +23,7 @@
   
  • 
   
   
    • 
   
  • 

From noreply at buildbot.pypy.org  Sun Aug 30 10:25:18 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Aug 2015 10:25:18 +0200 (CEST)
Subject: [pypy-commit] cffi default: Avoid importing other types and storing
 them in static global variables.
Message-ID: <20150830082518.100281C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2266:663cebc6e784
Date: 2015-08-30 10:25 +0200
http://bitbucket.org/cffi/cffi/changeset/663cebc6e784/

Log:	Avoid importing other types and storing them in static global
	variables.

	http://emptysqua.re/blog/python-c-extensions-and-mod-wsgi/#but-c
	-extensions-are-not-isolated

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6333,8 +6333,6 @@
         INITERROR;
 
 #if PY_MAJOR_VERSION >= 3
-    if (init_file_emulator() < 0)
-        INITERROR;
     return m;
 #endif
 }
diff --git a/c/file_emulator.h b/c/file_emulator.h
--- a/c/file_emulator.h
+++ b/c/file_emulator.h
@@ -1,21 +1,8 @@
 
 /* Emulation of PyFile_Check() and PyFile_AsFile() for Python 3. */
 
-static PyObject *PyIOBase_TypeObj;
 
-static int init_file_emulator(void)
-{
-    PyObject *io = PyImport_ImportModule("_io");
-    if (io == NULL)
-        return -1;
-    PyIOBase_TypeObj = PyObject_GetAttrString(io, "_IOBase");
-    if (PyIOBase_TypeObj == NULL)
-        return -1;
-    return 0;
-}
-
-
-#define PyFile_Check(p)  PyObject_IsInstance(p, PyIOBase_TypeObj)
+#define PyFile_Check(p)  PyObject_HasAttrString(p, "_checkSeekable")
 
 
 static void _close_file_capsule(PyObject *ob_capsule)

From noreply at buildbot.pypy.org  Sun Aug 30 10:27:55 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Aug 2015 10:27:55 +0200 (CEST)
Subject: [pypy-commit] cffi default: Backed out changeset 663cebc6e784
Message-ID: <20150830082755.9281B1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r2267:ad3140a30a7b
Date: 2015-08-30 10:28 +0200
http://bitbucket.org/cffi/cffi/changeset/ad3140a30a7b/

Log:	Backed out changeset 663cebc6e784

	_io._IOBase is actually a type written in C, so it should be shared
	among sub-interpreters anyway

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6333,6 +6333,8 @@
         INITERROR;
 
 #if PY_MAJOR_VERSION >= 3
+    if (init_file_emulator() < 0)
+        INITERROR;
     return m;
 #endif
 }
diff --git a/c/file_emulator.h b/c/file_emulator.h
--- a/c/file_emulator.h
+++ b/c/file_emulator.h
@@ -1,8 +1,21 @@
 
 /* Emulation of PyFile_Check() and PyFile_AsFile() for Python 3. */
 
+static PyObject *PyIOBase_TypeObj;
 
-#define PyFile_Check(p)  PyObject_HasAttrString(p, "_checkSeekable")
+static int init_file_emulator(void)
+{
+    PyObject *io = PyImport_ImportModule("_io");
+    if (io == NULL)
+        return -1;
+    PyIOBase_TypeObj = PyObject_GetAttrString(io, "_IOBase");
+    if (PyIOBase_TypeObj == NULL)
+        return -1;
+    return 0;
+}
+
+
+#define PyFile_Check(p)  PyObject_IsInstance(p, PyIOBase_TypeObj)
 
 
 static void _close_file_capsule(PyObject *ob_capsule)

From noreply at buildbot.pypy.org  Sun Aug 30 10:54:12 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Aug 2015 10:54:12 +0200 (CEST)
Subject: [pypy-commit] pypy default: No-op (hopefully): avoid using edi here
 if we can avoid it. I think it
Message-ID: <20150830085412.B30E11C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79290:2269d1da2105
Date: 2015-08-30 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/2269d1da2105/

Log:	No-op (hopefully): avoid using edi here if we can avoid it. I think
	it was safe anyway to do so at that point, but ecx is always a free
	reg and this makes future mistakes less likely.

diff --git a/rpython/jit/backend/x86/callbuilder.py b/rpython/jit/backend/x86/callbuilder.py
--- a/rpython/jit/backend/x86/callbuilder.py
+++ b/rpython/jit/backend/x86/callbuilder.py
@@ -266,9 +266,9 @@
                 rpy_errno = llerrno.get_rpy_errno_offset(self.asm.cpu)
             p_errno = llerrno.get_p_errno_offset(self.asm.cpu)
             tlofsreg = self.get_tlofs_reg()   # => esi or r12 (possibly reused)
-            mc.MOV_rm(edi.value, (tlofsreg.value, p_errno))
-            mc.MOV32_rm(edi.value, (edi.value, 0))
-            mc.MOV32_mr((tlofsreg.value, rpy_errno), edi.value)
+            mc.MOV_rm(ecx.value, (tlofsreg.value, p_errno))
+            mc.MOV32_rm(ecx.value, (ecx.value, 0))
+            mc.MOV32_mr((tlofsreg.value, rpy_errno), ecx.value)
 
         if handle_lasterror and (save_err & (rffi.RFFI_SAVE_LASTERROR |
                                              rffi.RFFI_SAVE_WSALASTERROR)):

From noreply at buildbot.pypy.org  Sun Aug 30 12:07:31 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 12:07:31 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some damage + force args
	early for preamble
Message-ID: <20150830100731.4F3521C145F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79291:45f5f8d0c2c8
Date: 2015-08-30 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/45f5f8d0c2c8/

Log:	fix some damage + force args early for preamble

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -473,11 +473,6 @@
             if indexb is None or indexb.contains(idx):
                 cf.force_lazy_setfield(self, None, can_cache)
 
-    def force_at_end_of_preamble(self):
-        self.cached_dict_reads.clear()
-        self.corresponding_array_descrs.clear()
-        self.force_all_lazy_setfields_and_arrayitems()
-
     def force_all_lazy_setfields_and_arrayitems(self):
         items = self.cached_fields.items()
         if not we_are_translated():
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -206,9 +206,6 @@
     def setup(self):
         pass
 
-    def force_at_end_of_preamble(self):
-        pass
-
     # Called after last operation has been propagated to flush out any posponed ops
     def flush(self):
         pass
@@ -296,10 +293,6 @@
             return
         self.replaces_guard[op] = value.last_guard_pos
 
-    def force_at_end_of_preamble(self):
-        for o in self.optimizations:
-            o.force_at_end_of_preamble()
-
     def force_box_for_end_of_preamble(self, box):
         if box.type == 'r':
             info = self.getptrinfo(box)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -343,7 +343,10 @@
 
 empty_info = EmptyInfo()
 
-class ShortPreambleBuilder(object):
+class AbstractShortPreambleBuilder(object):
+    pass
+
+class ShortPreambleBuilder(AbstractShortPreambleBuilder):
     """ ShortPreambleBuilder is used during optimizing of the peeled loop,
     starting from short_boxes exported from the preamble. It will build
     the short preamble and necessary extra label arguments
@@ -413,3 +416,6 @@
             TreeLoop.check_consistency_of(self.short_inputargs,
                                 self.short + [jump_op], check_descr=False)
         return [label_op] + self.short + [jump_op]
+
+class ExtendedShortPreambleBuilder(AbstractShortPreambleBuilder):
+    pass
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -4013,9 +4013,9 @@
         p3 = force_token()
         #
         p2 = new_with_vtable(descr=vref_descr)
+        setfield_gc(p2, p3, descr=virtualtokendescr)
+        setfield_gc(p2, NULL, descr=virtualforceddescr)
         setfield_gc(p0, p2, descr=nextdescr)
-        setfield_gc(p2, NULL, descr=virtualforceddescr)
-        setfield_gc(p2, p3, descr=virtualtokendescr)
         #
         call_may_force_n(i1, descr=mayforcevirtdescr)
         guard_not_forced() [p2, i1]
@@ -4024,9 +4024,9 @@
         setfield_gc(p2, NULL, descr=virtualtokendescr)
         p1 = new_with_vtable(descr=nodesize)
         p1b = new_with_vtable(descr=nodesize)
-        setfield_gc(p2, p1, descr=virtualforceddescr)
         setfield_gc(p1b, i1, descr=valuedescr)
         setfield_gc(p1, p1b, descr=nextdescr)
+        setfield_gc(p2, p1, descr=virtualforceddescr)
         jump(p0, i1)
         """
         # the point of this test is that 'i1' should show up in the fail_args
@@ -4686,8 +4686,8 @@
         [i0, p0]
         p1 = new_array(i0, descr=gcarraydescr)
         i1 = arraylen_gc(p1)
+        ifoo = arraylen_gc(p0, descr=gcarraydescr)
         setarrayitem_gc(p0, 0, p1, descr=gcarraydescr)
-        ifoo = arraylen_gc(p0, descr=gcarraydescr)
         jump(i0, p0)
         """
         self.optimize_loop(ops, expected)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -2,7 +2,7 @@
 import sys
 from rpython.jit.metainterp.history import Const, TargetToken, JitCellToken
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
-     ShortPreambleBuilder, PreambleOp
+     ShortPreambleBuilder, ExtendedShortPreambleBuilder, PreambleOp
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
      Optimization, LoopInfo, MININT, MAXINT
@@ -121,13 +121,15 @@
             flush=False)
         label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
         extra_same_as = self.short_preamble_producer.extra_same_as[:]
+        args = state.virtual_state.make_inputargs(
+            [self.get_box_replacement(x) for x in end_jump.getarglist()],
+            self.optimizer, force_boxes=True)
+        for arg in args:
+            self.optimizer.force_box(arg)
         target_token = self.finalize_short_preamble(label_op,
                                                     state.virtual_state)
         label_op.setdescr(target_token)
         # force the boxes for virtual state to match
-        state.virtual_state.make_inputargs(
-            [self.get_box_replacement(x) for x in end_jump.getarglist()],
-            self.optimizer, force_boxes=True)
         new_virtual_state = self.jump_to_existing_trace(end_jump)
         if new_virtual_state is not None:
             celltoken = start_label.getdescr()
@@ -153,7 +155,7 @@
             return self.jump_to_preamble(cell_token, jump_op, info)
         # force all the information that does not go to the short
         # preamble at all
-        self.optimizer.force_at_end_of_preamble()
+        self.optimizer.flush()
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
         vs = self.jump_to_existing_trace(jump_op)
@@ -196,7 +198,7 @@
         target_token.virtual_state = virtual_state
         target_token.short_preamble = short_preamble
         jitcelltoken.target_tokens.append(target_token)
-        self.short_preamble_producer = None # no more boxes
+        self.short_preamble_producer = ExtendedShortPreambleBuilder()
         label_op.initarglist(label_op.getarglist() + sb.used_boxes)
         return target_token
 
@@ -275,7 +277,7 @@
     def export_state(self, start_label, original_label_args, renamed_inputargs):
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
-        self.optimizer.force_at_end_of_preamble()
+        self.optimizer.flush()
         virtual_state = self.get_virtual_state(end_args)
         end_args = [self.get_box_replacement(arg) for arg in end_args]
         infos = {}

From noreply at buildbot.pypy.org  Sun Aug 30 12:08:07 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 12:08:07 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: kill some long dead code
Message-ID: <20150830100807.C86301C145F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79292:df3ba689cbaf
Date: 2015-08-30 12:08 +0200
http://bitbucket.org/pypy/pypy/changeset/df3ba689cbaf/

Log:	kill some long dead code

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -59,40 +59,6 @@
             structbox = optimizer.get_box_replacement(self.cached_structs[i])
             info.produce_short_preamble_ops(structbox, descr, index, optimizer,
                                             shortboxes)
-        return
-
-        XXXXX
-        for structvalue in self._cached_fields_getfield_op.keys():
-            op = self._cached_fields_getfield_op[structvalue]
-            if not op:
-                continue
-            value = optimizer.getvalue(op.getarg(0))
-            if value in optimizer.opaque_pointers:
-                if value.getlevel() < LEVEL_KNOWNCLASS:
-                    continue
-                if op.getopnum() != rop.SETFIELD_GC and op.getopnum() != rop.GETFIELD_GC:
-                    continue
-            if structvalue in self._cached_fields:
-                if op.getopnum() == rop.SETFIELD_GC:
-                    result = op.getarg(1)
-                    if isinstance(result, Const):
-                        newresult = result.clonebox()
-                        optimizer.make_constant(newresult, result)
-                        result = newresult
-                    getop = ResOperation(rop.GETFIELD_GC, [op.getarg(0)],
-                                         result, op.getdescr())
-                    shortboxes.add_potential(getop, synthetic=True)
-                if op.getopnum() == rop.SETARRAYITEM_GC:
-                    result = op.getarg(2)
-                    if isinstance(result, Const):
-                        newresult = result.clonebox()
-                        optimizer.make_constant(newresult, result)
-                        result = newresult
-                    getop = ResOperation(rop.GETARRAYITEM_GC, [op.getarg(0), op.getarg(1)],
-                                         result, op.getdescr())
-                    shortboxes.add_potential(getop, synthetic=True)
-                elif op.result is not None:
-                    shortboxes.add_potential(op)
 
     def possible_aliasing(self, optheap, opinfo):
         # If lazy_setfield is set and contains a setfield on a different

From noreply at buildbot.pypy.org  Sun Aug 30 12:12:10 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 12:12:10 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix wrong optimization
Message-ID: <20150830101210.85EA41C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79293:93678a196e93
Date: 2015-08-30 12:12 +0200
http://bitbucket.org/pypy/pypy/changeset/93678a196e93/

Log:	fix wrong optimization

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -186,8 +186,7 @@
                 subbox = optforce.force_box(fld)
                 setfieldop = ResOperation(rop.SETFIELD_GC, [op, subbox],
                                           descr=flddescr)
-                if not flddescr.is_always_pure():
-                    self._fields[i] = None
+                self._fields[i] = None
                 optforce.emit_operation(setfieldop)
 
     def _force_at_the_end_of_preamble(self, op, optforce, rec):
@@ -453,8 +452,7 @@
             setop = ResOperation(rop.SETARRAYITEM_GC,
                                  [op, ConstInt(i), subbox],
                                   descr=arraydescr)
-            if not self.arraydescr.is_always_pure():
-                self._items[i] = None
+            self._items[i] = None
             optforce.emit_operation(setop)
         optforce.pure_from_args(rop.ARRAYLEN_GC, [op], ConstInt(len(self._items)))
 

From noreply at buildbot.pypy.org  Sun Aug 30 12:15:01 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 12:15:01 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix one more order of things
Message-ID: <20150830101501.C70131C03C0@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79294:ac94fd7b5f0e
Date: 2015-08-30 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/ac94fd7b5f0e/

Log:	fix one more order of things

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -120,12 +120,12 @@
             start_label.getarglist()[:], ops, call_pure_results, False,
             flush=False)
         label_op = ResOperation(rop.LABEL, label_args, start_label.getdescr())
-        extra_same_as = self.short_preamble_producer.extra_same_as[:]
         args = state.virtual_state.make_inputargs(
             [self.get_box_replacement(x) for x in end_jump.getarglist()],
             self.optimizer, force_boxes=True)
         for arg in args:
             self.optimizer.force_box(arg)
+        extra_same_as = self.short_preamble_producer.extra_same_as[:]
         target_token = self.finalize_short_preamble(label_op,
                                                     state.virtual_state)
         label_op.setdescr(target_token)

From noreply at buildbot.pypy.org  Sun Aug 30 12:20:27 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Aug 2015 12:20:27 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: Missing get_type_id() methods
	here
Message-ID: <20150830102027.360121C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79295:b61e2bd68f68
Date: 2015-08-30 12:20 +0200
http://bitbucket.org/pypy/pypy/changeset/b61e2bd68f68/

Log:	Missing get_type_id() methods here

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -63,6 +63,10 @@
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self.vtable))
 
+    def get_type_id(self):
+        assert self.tid != 0
+        return self.tid
+
 def get_size_descr(gccache, STRUCT, vtable):
     cache = gccache._cache_size
     assert not isinstance(vtable, bool)
@@ -288,7 +292,9 @@
 
         assert False
 
-
+    def get_type_id(self):
+        assert self.tid != 0
+        return self.tid
 
     def repr_of_descr(self):
         return '' % (self.flag, self.itemsize)

From noreply at buildbot.pypy.org  Sun Aug 30 12:26:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Aug 2015 12:26:49 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: 'supports_guard_gc_type' was
	bogus
Message-ID: <20150830102649.86DCF1C03C0@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: optresult-unroll
Changeset: r79296:b8e5e200c5c4
Date: 2015-08-30 12:27 +0200
http://bitbucket.org/pypy/pypy/changeset/b8e5e200c5c4/

Log:	'supports_guard_gc_type' was bogus

diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -253,6 +253,7 @@
     str_type_id           = 0
     unicode_type_id       = 0
     get_malloc_slowpath_addr = None
+    supports_guard_gc_type   = False
 
     def is_shadow_stack(self):
         return False
@@ -417,6 +418,7 @@
     kind = 'framework'
     round_up = True
     layoutbuilder = None
+    supports_guard_gc_type = True
 
     def is_shadow_stack(self):
         return self.gcrootmap.is_shadow_stack
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -40,7 +40,7 @@
         else:
             translator = None
         self.gc_ll_descr = get_ll_description(gcdescr, translator, rtyper)
-        self.supports_guard_gc_type = bool(translate_support_code)
+        self.supports_guard_gc_type = self.gc_ll_descr.supports_guard_gc_type
         if translator and translator.config.translation.gcremovetypeptr:
             self.vtable_offset = None
         else:

From noreply at buildbot.pypy.org  Sun Aug 30 19:54:10 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Aug 2015 19:54:10 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: remove ineffective call for
	contributions
Message-ID: <20150830175410.10BAA1C0F47@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79297:1f367e4eb974
Date: 2015-08-30 20:53 +0300
http://bitbucket.org/pypy/pypy/changeset/1f367e4eb974/

Log:	remove ineffective call for contributions

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -12,27 +12,9 @@
     http://pypy.org/download.html
 
 We would like to thank our donors for the continued support of the PyPy
-project, and for those who donate to our three sub-projects, as well as our
-volunteers and contributors.  
-
-We've shown quite a bit of progress, but we're slowly running out of funds.
-Please consider donating more, or even better convince your employer to donate,
-so we can finish those projects! The three sub-projects are:
-
-* `Py3k`_ (supporting Python 3.x): We have released a Python 3.2.5 compatible version
-  we call PyPy3 2.4.0, and are working toward a Python 3.3 compatible version
-
-* `STM`_ (software transactional memory): We have released a first working version,
-  and continue to try out new promising paths of achieving a fast multithreaded Python
-
-* `NumPy`_ which requires installation of our fork of upstream numpy,
-  available `on bitbucket`_
+project, and our volunteers and contributors.  
 
 .. _`cffi`: https://cffi.readthedocs.org
-.. _`Py3k`: http://pypy.org/py3donate.html
-.. _`STM`: http://pypy.org/tmdonate2.html
-.. _`NumPy`: http://pypy.org/numpydonate.html
-.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy
 
 We would also like to encourage new people to join the project. PyPy has many
 layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation

From noreply at buildbot.pypy.org  Sun Aug 30 20:01:28 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Aug 2015 20:01:28 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: document vmprof support for OS X
Message-ID: <20150830180128.EA2F61C0FD4@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79298:3e656718275b
Date: 2015-08-30 21:01 +0300
http://bitbucket.org/pypy/pypy/changeset/3e656718275b/

Log:	document vmprof support for OS X

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -73,7 +73,7 @@
 
   * The python stdlib was updated to 2.7.10 from 2.7.9
 
-  * vmprof now supports multiple threads
+  * vmprof now supports multiple threads and OS X
 
   * The translation process builds cffi import libraries for some stdlib
     packages, which should prevent confusion when package.py is not used

From noreply at buildbot.pypy.org  Sun Aug 30 20:49:16 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Sun, 30 Aug 2015 20:49:16 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: Also kill the call for Windows 64,
 which has been here for many years
Message-ID: <20150830184916.97A511C0507@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: release-2.6.x
Changeset: r79299:328731dba74c
Date: 2015-08-30 20:49 +0200
http://bitbucket.org/pypy/pypy/changeset/328731dba74c/

Log:	Also kill the call for Windows 64, which has been here for many
	years with no effect

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -34,18 +34,15 @@
 due to its integrated tracing JIT compiler.
 
 This release supports **x86** machines on most common operating systems
-(Linux 32/64, Mac OS X 64, Windows, OpenBSD_, freebsd_),
+(Linux 32/64, Mac OS X 64, Windows 32, OpenBSD_, freebsd_),
 as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux.
 
-While we support 32 bit python on Windows, work on the native Windows 64
-bit python is still stalling, we would welcome a volunteer 
-to `handle that`_. We also welcome developers with other operating systems or
+We also welcome developers of other
 `dynamic languages`_ to see what RPython can do for them.
 
 .. _`pypy and cpython 2.7.x`: http://speed.pypy.org
 .. _OpenBSD: http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/lang/pypy
 .. _freebsd: https://svnweb.freebsd.org/ports/head/lang/pypy/
-.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation
 .. _`dynamic languages`: http://pypyjs.org
 
 Highlights 

From noreply at buildbot.pypy.org  Sun Aug 30 21:45:32 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Aug 2015 21:45:32 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: fix link
Message-ID: <20150830194532.F3F1B1C145F@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79300:e9009e0d30b7
Date: 2015-08-30 22:44 +0300
http://bitbucket.org/pypy/pypy/changeset/e9009e0d30b7/

Log:	fix link

diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
--- a/pypy/doc/release-2.6.1.rst
+++ b/pypy/doc/release-2.6.1.rst
@@ -116,7 +116,7 @@
     one sequence argument
 
 .. _`vmprof`: https://vmprof.readthedocs.org
-.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.0.html
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.1.html
 
 Please try it out and let us know what you think. We welcome
 success stories, `experiments`_,  or `benchmarks`_, we know you are using PyPy, please tell us about it!

From noreply at buildbot.pypy.org  Sun Aug 30 21:51:19 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Sun, 30 Aug 2015 21:51:19 +0200 (CEST)
Subject: [pypy-commit] pypy release-2.6.x: update readthedocs configuration
Message-ID: <20150830195119.AED671C149C@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: release-2.6.x
Changeset: r79301:1c0058ccbe23
Date: 2015-08-30 22:51 +0300
http://bitbucket.org/pypy/pypy/changeset/1c0058ccbe23/

Log:	update readthedocs configuration

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -67,7 +67,7 @@
 # The short X.Y version.
 version = '2.6'
 # The full version, including alpha/beta/rc tags.
-release = '2.6.0'
+release = '2.6.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py
--- a/rpython/doc/conf.py
+++ b/rpython/doc/conf.py
@@ -68,7 +68,7 @@
 # The short X.Y version.
 version = '2.6'
 # The full version, including alpha/beta/rc tags.
-release = '2.6.0'
+release = '2.6.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

From noreply at buildbot.pypy.org  Sun Aug 30 22:52:56 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 22:52:56 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150830205256.ADE411C0627@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79302:9d930c5c8569
Date: 2015-08-30 12:22 +0200
http://bitbucket.org/pypy/pypy/changeset/9d930c5c8569/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/test/test_jitdriver.py b/rpython/jit/metainterp/test/test_jitdriver.py
--- a/rpython/jit/metainterp/test/test_jitdriver.py
+++ b/rpython/jit/metainterp/test/test_jitdriver.py
@@ -104,7 +104,7 @@
         res = self.meta_interp(loop2, [4, 40], repeat=7, inline=True)
         assert res == loop2(4, 40)
         # we expect no int_sub, but a residual call
-        self.check_resops(call=2, int_sub=0)
+        self.check_resops(call_i=2, int_sub=0)
 
     def test_multiple_jits_trace_too_long(self):
         myjitdriver1 = JitDriver(greens=["n"], reds=["i", "box"])
diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -8,7 +8,8 @@
 
     def check_all_virtualized(self):
         self.check_resops(setarrayitem_gc=0, new_array=0, arraylen_gc=0,
-                          getarrayitem_gc=0)        
+                          getarrayitem_gc_i=0, getarrayitem_gc_r=0,
+                          getarrayitem_gc_f=0)        
 
     def test_simple_array(self):
         jitdriver = JitDriver(greens = [], reds = ['n'])
@@ -57,7 +58,7 @@
         res = self.meta_interp(f, [10], listops=True)
         assert res == f(10)
         # one setitem should be gone by now
-        self.check_resops(setarrayitem_gc=4, getarrayitem_gc=2, call=2)
+        self.check_resops(setarrayitem_gc=4, getarrayitem_gc_i=2, call_r=2)
 
 
     def test_ll_fixed_setitem_fast(self):
@@ -95,7 +96,7 @@
 
         res = self.meta_interp(f, [10], listops=True, backendopt=True)
         assert res == f(10)
-        self.check_resops(setarrayitem_gc=0, call=0, getarrayitem_gc=0)
+        self.check_resops(setarrayitem_gc=0, call=0, getarrayitem_gc_i=0)
 
     def test_arraycopy_simpleoptimize(self):
         def f():
@@ -361,7 +362,7 @@
         assert res == f(37)
         # There is the one actual field on a, plus several fields on the list
         # itself
-        self.check_resops(getfield_gc=7)
+        self.check_resops(getfield_gc_i=2, getfield_gc_r=5)
 
     def test_conditional_call_append(self):
         jitdriver = JitDriver(greens = [], reds = 'auto')

From noreply at buildbot.pypy.org  Sun Aug 30 22:52:58 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 22:52:58 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some more tests
Message-ID: <20150830205258.B23C31C0627@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79303:ed35f8e083e4
Date: 2015-08-30 12:26 +0200
http://bitbucket.org/pypy/pypy/changeset/ed35f8e083e4/

Log:	fix some more tests

diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -1023,7 +1023,7 @@
         res = self.meta_interp(portal, [2, 0], inline=True,
                                policy=StopAtXPolicy(residual))
         assert res == portal(2, 0)
-        self.check_resops(call_assembler=4)
+        self.check_resops(call_assembler_i=4)
 
     def test_inline_without_hitting_the_loop(self):
         driver = JitDriver(greens = ['codeno'], reds = ['i'],
@@ -1073,7 +1073,7 @@
         assert portal(0, 1) == 2095
         res = self.meta_interp(portal, [0, 1], inline=True)
         assert res == 2095
-        self.check_resops(call_assembler=12)
+        self.check_resops(call_assembler_i=12)
 
     def test_inline_with_hitting_the_loop_sometimes_exc(self):
         driver = JitDriver(greens = ['codeno'], reds = ['i', 'k'],
@@ -1111,7 +1111,7 @@
         assert main(0, 1) == 2095
         res = self.meta_interp(main, [0, 1], inline=True)
         assert res == 2095
-        self.check_resops(call_assembler=12)
+        self.check_resops(call_assembler_i=12)
 
     def test_inline_recursion_limit(self):
         driver = JitDriver(greens = ["threshold", "loop"], reds=["i"])

From noreply at buildbot.pypy.org  Sun Aug 30 22:53:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 22:53:00 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: merge
Message-ID: <20150830205300.DE4731C0627@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79304:1828347b4ed8
Date: 2015-08-30 22:53 +0200
http://bitbucket.org/pypy/pypy/changeset/1828347b4ed8/

Log:	merge

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -63,6 +63,10 @@
     def get_vtable(self):
         return heaptracker.adr2int(llmemory.cast_ptr_to_adr(self.vtable))
 
+    def get_type_id(self):
+        assert self.tid != 0
+        return self.tid
+
 def get_size_descr(gccache, STRUCT, vtable):
     cache = gccache._cache_size
     assert not isinstance(vtable, bool)
@@ -288,7 +292,9 @@
 
         assert False
 
-
+    def get_type_id(self):
+        assert self.tid != 0
+        return self.tid
 
     def repr_of_descr(self):
         return '' % (self.flag, self.itemsize)
diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py
--- a/rpython/jit/backend/llsupport/gc.py
+++ b/rpython/jit/backend/llsupport/gc.py
@@ -253,6 +253,7 @@
     str_type_id           = 0
     unicode_type_id       = 0
     get_malloc_slowpath_addr = None
+    supports_guard_gc_type   = False
 
     def is_shadow_stack(self):
         return False
@@ -417,6 +418,7 @@
     kind = 'framework'
     round_up = True
     layoutbuilder = None
+    supports_guard_gc_type = True
 
     def is_shadow_stack(self):
         return self.gcrootmap.is_shadow_stack
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -40,7 +40,7 @@
         else:
             translator = None
         self.gc_ll_descr = get_ll_description(gcdescr, translator, rtyper)
-        self.supports_guard_gc_type = bool(translate_support_code)
+        self.supports_guard_gc_type = self.gc_ll_descr.supports_guard_gc_type
         if translator and translator.config.translation.gcremovetypeptr:
             self.vtable_offset = None
         else:

From noreply at buildbot.pypy.org  Sun Aug 30 23:17:39 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 23:17:39 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: pass test_quasiimmut
Message-ID: <20150830211739.21C0F1C0507@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79305:4a71f8c4189d
Date: 2015-08-30 23:17 +0200
http://bitbucket.org/pypy/pypy/changeset/4a71f8c4189d/

Log:	pass test_quasiimmut

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -220,6 +220,8 @@
     loop = create_empty_loop(metainterp)
     loop.original_jitcell_token = jitcell_token
     loop.inputargs = loop_info.inputargs
+    if loop_info.quasi_immutable_deps:
+        loop.quasi_immutable_deps = loop_info.quasi_immutable_deps
     jump_op = ops[-1]
     target_token = TargetToken(jitcell_token)
     target_token.original_jitcell_token = jitcell_token
@@ -287,6 +289,13 @@
     loop = create_empty_loop(metainterp)
     loop.original_jitcell_token = jitcell_token
     loop.inputargs = start_state.renamed_inputargs
+    quasi_immutable_deps = {}
+    if start_state.quasi_immutable_deps:
+        quasi_immutable_deps.update(start_state.quasi_immutable_deps)
+    if loop_info.quasi_immutable_deps:
+        quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
+    if quasi_immutable_deps:
+        loop.quasi_immutable_deps = quasi_immutable_deps
     start_descr = TargetToken(jitcell_token,
                               original_jitcell_token=jitcell_token)
     start_label = ResOperation(rop.LABEL, start_state.renamed_inputargs,
@@ -335,17 +344,18 @@
         return None
 
     loop = partial_trace
+    loop.original_jitcell_token = loop_jitcell_token
     loop.operations = (loop.operations + loop_info.extra_same_as +
                        [loop_info.label_op]
                        + loop_ops)
 
-    #quasi_immutable_deps = {}
-    #if loop.quasi_immutable_deps:
-    #    quasi_immutable_deps.update(loop.quasi_immutable_deps)
-    #if part.quasi_immutable_deps:
-    #    quasi_immutable_deps.update(part.quasi_immutable_deps)
-    #if quasi_immutable_deps:
-    #    loop.quasi_immutable_deps = quasi_immutable_deps
+    quasi_immutable_deps = {}
+    if loop_info.quasi_immutable_deps:
+        quasi_immutable_deps.update(loop_info.quasi_immutable_deps)
+    if start_state.quasi_immutable_deps:
+        quasi_immutable_deps.update(start_state.quasi_immutable_deps)
+    if quasi_immutable_deps:
+        loop.quasi_immutable_deps = quasi_immutable_deps
 
     target_token = loop.operations[-1].getdescr()
     resumekey.compile_and_attach(metainterp, loop, inputargs)
@@ -420,13 +430,12 @@
         emit_op(extra_ops, op)
     loop.operations = extra_ops
 
-#def propagate_original_jitcell_token(trace):
-#    for op in trace.operations:
-#        if op.getopnum() == rop.LABEL:
-#            token = op.getdescr()
-#            assert isinstance(token, TargetToken)
-#            assert token.original_jitcell_token is None
-#            token.original_jitcell_token = trace.original_jitcell_token
+def propagate_original_jitcell_token(trace):
+    for op in trace.operations:
+        if op.getopnum() == rop.LABEL:
+            token = op.getdescr()
+            assert isinstance(token, TargetToken)
+            token.original_jitcell_token = trace.original_jitcell_token
 
 
 def do_compile_loop(jd_id, unique_id, metainterp_sd, inputargs, operations,
@@ -767,7 +776,7 @@
         inputargs = new_loop.inputargs
         if not we_are_translated():
             self._debug_suboperations = new_loop.operations
-        #propagate_original_jitcell_token(new_loop)
+        propagate_original_jitcell_token(new_loop)
         send_bridge_to_backend(metainterp.jitdriver_sd, metainterp.staticdata,
                                self, inputargs, new_loop.operations,
                                new_loop.original_jitcell_token)
@@ -954,7 +963,7 @@
         metainterp_sd = metainterp.staticdata
         jitdriver_sd = metainterp.jitdriver_sd
         new_loop.original_jitcell_token = jitcell_token = make_jitcell_token(jitdriver_sd)
-        #propagate_original_jitcell_token(new_loop)
+        propagate_original_jitcell_token(new_loop)
         send_loop_to_backend(self.original_greenkey, metainterp.jitdriver_sd,
                              metainterp_sd, new_loop, "entry bridge",
                              orig_inputargs)
@@ -1015,6 +1024,8 @@
 
     new_trace = create_empty_loop(metainterp)
     new_trace.operations = newops
+    if info.quasi_immutable_deps:
+        new_trace.quasi_immutable_deps = info.quasi_immutable_deps
     if info.final():
         new_trace.inputargs = info.inputargs
         target_token = new_trace.operations[-1].getdescr()
diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py
--- a/rpython/jit/metainterp/history.py
+++ b/rpython/jit/metainterp/history.py
@@ -782,6 +782,15 @@
         if 'getfield_gc' in check:
             assert check.pop('getfield_gc') == 0
             check['getfield_gc_i'] = check['getfield_gc_r'] = check['getfield_gc_f'] = 0
+        if 'getfield_gc_pure' in check:
+            assert check.pop('getfield_gc_pure') == 0
+            check['getfield_gc_pure_i'] = check['getfield_gc_pure_r'] = check['getfield_gc_pure_f'] = 0
+        if 'getarrayitem_gc_pure' in check:
+            assert check.pop('getarrayitem_gc_pure') == 0
+            check['getarrayitem_gc_pure_i'] = check['getarrayitem_gc_pure_r'] = check['getarrayitem_gc_pure_f'] = 0            
+        if 'getarrayitem_gc' in check:
+            assert check.pop('getarrayitem_gc') == 0
+            check['getarrayitem_gc_i'] = check['getarrayitem_gc_r'] = check['getarrayitem_gc_f'] = 0
         for loop in self.get_all_loops():
             insns = loop.summary(adding_insns=insns, omit_finish=omit_finish)
         return self._check_insns(insns, expected, check)
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -24,8 +24,9 @@
     pass
 
 class BasicLoopInfo(LoopInfo):
-    def __init__(self, inputargs):
+    def __init__(self, inputargs, quasi_immutable_deps):
         self.inputargs = inputargs
+        self.quasi_immutable_deps = quasi_immutable_deps
 
     def final(self):
         return True
@@ -508,15 +509,14 @@
         for i in range(last):
             self._really_emitted_operation = None
             self.first_optimization.propagate_forward(ops[i])
-        #self.loop.operations = self.get_newoperations()
-        #self.loop.quasi_immutable_deps = self.quasi_immutable_deps
         # accumulate counters
         if flush:
             self.flush()
         if extra_jump:
             self.first_optimization.propagate_forward(ops[-1])
         self.resumedata_memo.update_counters(self.metainterp_sd.profiler)
-        return BasicLoopInfo(newargs), self._newoperations
+        return (BasicLoopInfo(newargs, self.quasi_immutable_deps),
+                self._newoperations)
 
     def _clean_optimization_info(self, lst):
         for op in lst:
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -105,6 +105,7 @@
             flush=False)
         exported_state = self.export_state(start_label, end_label.getarglist(),
                                            info.inputargs)
+        exported_state.quasi_immutable_deps = info.quasi_immutable_deps
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
@@ -138,7 +139,8 @@
             return (UnrollInfo(target_token, label_op, []),
                     self.optimizer._newoperations)
             #return new_virtual_state, self.optimizer._newoperations
-        return (UnrollInfo(target_token, label_op, extra_same_as),
+        return (UnrollInfo(target_token, label_op, extra_same_as,
+                           self.optimizer.quasi_immutable_deps),
                 self.optimizer._newoperations)
 
     def optimize_bridge(self, start_label, operations, call_pure_results,
@@ -181,6 +183,7 @@
         exported_state = self.export_state(start_label,
                                            operations[-1].getarglist(),
                                            info.inputargs)
+        exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
@@ -198,7 +201,7 @@
         target_token.virtual_state = virtual_state
         target_token.short_preamble = short_preamble
         jitcelltoken.target_tokens.append(target_token)
-        self.short_preamble_producer = ExtendedShortPreambleBuilder()
+        self.short_preamble_producer = None # ExtendedShortPreambleBuilder()
         label_op.initarglist(label_op.getarglist() + sb.used_boxes)
         return target_token
 
@@ -350,10 +353,12 @@
     * label_args - label operations at the beginning
     * extra_same_as - list of extra same as to add at the end of the preamble
     """
-    def __init__(self, target_token, label_op, extra_same_as):
+    def __init__(self, target_token, label_op, extra_same_as,
+                 quasi_immutable_deps):
         self.target_token = target_token
         self.label_op = label_op
         self.extra_same_as = extra_same_as
+        self.quasi_immutable_deps = quasi_immutable_deps
 
     def final(self):
         return True
@@ -370,6 +375,7 @@
     * short boxes - a mapping op -> preamble_op
     * renamed_inputargs - the start label arguments in optimized version
     * short_inputargs - the renamed inputargs for short preamble
+    * quasi_immutable_deps - for tracking quasi immutables
     """
     
     def __init__(self, end_args, next_iteration_args, virtual_state,
diff --git a/rpython/jit/metainterp/test/test_quasiimmut.py b/rpython/jit/metainterp/test/test_quasiimmut.py
--- a/rpython/jit/metainterp/test/test_quasiimmut.py
+++ b/rpython/jit/metainterp/test/test_quasiimmut.py
@@ -124,7 +124,8 @@
         assert f(100, 7) == 721
         res = self.meta_interp(f, [100, 7])
         assert res == 721
-        self.check_resops(guard_not_invalidated=0, getfield_gc=1, getfield_gc_pure=2)
+        self.check_resops(guard_not_invalidated=0, getfield_gc_r=1,
+                          getfield_gc_pure_i=2)
         #
         from rpython.jit.metainterp.warmspot import get_stats
         loops = get_stats().loops
@@ -369,8 +370,12 @@
         #
         res = self.meta_interp(f, [100, 7])
         assert res == 700
-        self.check_resops(getarrayitem_gc_pure=0, guard_not_invalidated=2,
-                          getarrayitem_gc=0, getfield_gc=0, getfield_gc_pure=0)
+        self.check_resops(getarrayitem_gc_pure_i=0, guard_not_invalidated=2,
+                          getarrayitem_gc_pure_r=0,
+                          getarrayitem_gc_i=0,
+                          getarrayitem_gc_r=0,
+                          getfield_gc_i=0, getfield_gc_pure_i=0,
+                          getfield_gc_r=0, getfield_gC_pure_r=0)
         #
         from rpython.jit.metainterp.warmspot import get_stats
         loops = get_stats().loops
@@ -399,8 +404,10 @@
         res = self.meta_interp(f, [100, 7], enable_opts="")
         assert res == 700
         # operations must have been removed by the frontend
-        self.check_resops(getarrayitem_gc_pure=0, guard_not_invalidated=1,
-                          getarrayitem_gc=0, getfield_gc=0, getfield_gc_pure=0)
+        self.check_resops(getarrayitem_gc_pure_i=0, guard_not_invalidated=1,
+                          getarrayitem_gc_i=0,
+                          getfield_gc=0, getfield_gc_pure_i=0,
+                          getfield_gc_pure_r=0)
 
     def test_list_length_1(self):
         myjitdriver = JitDriver(greens=['foo'], reds=['x', 'total'])

From noreply at buildbot.pypy.org  Sun Aug 30 23:37:56 2015
From: noreply at buildbot.pypy.org (alex_gaynor)
Date: Sun, 30 Aug 2015 23:37:56 +0200 (CEST)
Subject: [pypy-commit] pypy default: bumped greenlet version -- nothing is
 required to get to 0.4.9
Message-ID: <20150830213756.5A8C51C0627@cobra.cs.uni-duesseldorf.de>

Author: Alex Gaynor 
Branch: 
Changeset: r79306:9c171619ae6d
Date: 2015-08-30 17:38 -0400
http://bitbucket.org/pypy/pypy/changeset/9c171619ae6d/

Log:	bumped greenlet version -- nothing is required to get to 0.4.9

diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.7
+Version: 0.4.9
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.7"
+__version__ = "0.4.9"
 
 # ____________________________________________________________
 # Exceptions

From noreply at buildbot.pypy.org  Sun Aug 30 23:51:03 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 23:51:03 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: disable max_guards apparently
 I have no idea what it is doing
Message-ID: <20150830215103.6E4CF1C149C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79307:4b8a4aee4962
Date: 2015-08-30 23:51 +0200
http://bitbucket.org/pypy/pypy/changeset/4b8a4aee4962/

Log:	disable max_guards apparently I have no idea what it is doing

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -171,15 +171,12 @@
         else:
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
-        maxguards = warmrunnerdescr.memory_manager.max_retrace_guards
-        guard_count = 0
-        for op in self.optimizer._newoperations:
-            if op.is_guard():
-                guard_count += 1
-        if guard_count > maxguards:
-            target_token = cell_token.target_tokens[0]
-            target_token.targeting_jitcell_token.retraced_count = sys.maxint
-            return self.jump_to_preamble(cell_token, jump_op, info)
+        #maxguards = warmrunnerdescr.memory_manager.max_retrace_guards
+        #guard_count = self.count_guards(self.optimizer._newoperations)
+        #if guard_count > maxguards:
+        #    target_token = cell_token.target_tokens[0]
+        #    target_token.targeting_jitcell_token.retraced_count = sys.maxint
+        #    return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(start_label,
                                            operations[-1].getarglist(),
                                            info.inputargs)
@@ -345,6 +342,13 @@
             return effectinfo.extraeffect != effectinfo.EF_ELIDABLE_CANNOT_RAISE
         return False
 
+    def count_guards(self, ops):
+        guard_count = 0
+        for op in ops:
+            if op.is_guard():
+                guard_count += 1
+        return guard_count
+
 
 class UnrollInfo(LoopInfo):
     """ A state after optimizing the peeled loop, contains the following:
diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py
--- a/rpython/jit/metainterp/test/test_recursive.py
+++ b/rpython/jit/metainterp/test/test_recursive.py
@@ -1111,7 +1111,7 @@
         assert main(0, 1) == 2095
         res = self.meta_interp(main, [0, 1], inline=True)
         assert res == 2095
-        self.check_resops(call_assembler_i=12)
+        self.check_resops(call_assembler_n=12)
 
     def test_inline_recursion_limit(self):
         driver = JitDriver(greens = ["threshold", "loop"], reds=["i"])
@@ -1137,7 +1137,7 @@
         res1 = portal(10, True, 0)
         res2 = self.meta_interp(portal, [10, True, 0], inline=True)
         assert res1 == res2
-        self.check_resops(call_assembler=2)
+        self.check_resops(call_assembler_i=2)
 
         res1 = portal(9, True, 0)
         res2 = self.meta_interp(portal, [9, True, 0], inline=True)

From noreply at buildbot.pypy.org  Sun Aug 30 23:55:00 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Sun, 30 Aug 2015 23:55:00 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: ups
Message-ID: <20150830215500.EE3931C149C@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79308:7064fdf63f12
Date: 2015-08-30 23:55 +0200
http://bitbucket.org/pypy/pypy/changeset/7064fdf63f12/

Log:	ups

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -136,7 +136,8 @@
             celltoken = start_label.getdescr()
             assert isinstance(celltoken, JitCellToken)
             self.jump_to_preamble(celltoken, end_jump, info)
-            return (UnrollInfo(target_token, label_op, []),
+            return (UnrollInfo(target_token, label_op, [],
+                               self.optimizer.quasi_immutable_deps),
                     self.optimizer._newoperations)
             #return new_virtual_state, self.optimizer._newoperations
         return (UnrollInfo(target_token, label_op, extra_same_as,

From noreply at buildbot.pypy.org  Mon Aug 31 09:33:42 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 09:33:42 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix some tests
Message-ID: <20150831073342.D54481C145F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79309:bf7a0a3b5f92
Date: 2015-08-31 09:33 +0200
http://bitbucket.org/pypy/pypy/changeset/bf7a0a3b5f92/

Log:	fix some tests

diff --git a/rpython/jit/metainterp/test/test_send.py b/rpython/jit/metainterp/test/test_send.py
--- a/rpython/jit/metainterp/test/test_send.py
+++ b/rpython/jit/metainterp/test/test_send.py
@@ -42,7 +42,7 @@
         assert res == 2
         # 'len' becomes a getfield('num_items') for now in lltype,
         # which is itself encoded as a 'getfield_gc'
-        self.check_resops(call=2, getfield_gc=2)
+        self.check_resops(call_r=2, getfield_gc_i=2)
 
     def test_send_to_single_target_method(self):
         myjitdriver = JitDriver(greens = [], reds = ['i', 'counter'])
@@ -66,9 +66,9 @@
         res = self.meta_interp(f, [1], policy=StopAtXPolicy(externfn),
                                backendopt=True)
         assert res == 43
-        self.check_resops({'int_gt': 2, 'getfield_gc': 2,
+        self.check_resops({'int_gt': 2, 'getfield_gc_i': 2,
                            'guard_true': 2, 'int_sub': 2, 'jump': 1,
-                           'call': 2, 'guard_no_exception': 2,
+                           'call_r': 2, 'guard_no_exception': 2,
                            'int_add': 2})
 
     def test_red_send_to_green_receiver(self):
@@ -557,7 +557,7 @@
         policy = StopAtXPolicy(new, A.foo.im_func, B.foo.im_func)
         res = self.meta_interp(fn, [0, 20], policy=policy)
         assert res == 42
-        self.check_resops(call=2)
+        self.check_resops(call_i=2)
 
 
     def test_residual_oosend_with_void(self):
@@ -585,7 +585,7 @@
         policy = StopAtXPolicy(new, A.foo.im_func)
         res = self.meta_interp(fn, [1, 20], policy=policy)
         assert res == 41
-        self.check_resops(call=2)
+        self.check_resops(call_i=2)
 
     def test_constfold_pure_oosend(self):
         myjitdriver = JitDriver(greens=[], reds = ['i', 'obj'])

From noreply at buildbot.pypy.org  Mon Aug 31 09:36:34 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 31 Aug 2015 09:36:34 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: slicing index must be positive
Message-ID: <20150831073634.6ADA21C152B@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79310:51a6240fbce7
Date: 2015-08-31 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/51a6240fbce7/

Log:	slicing index must be positive adding split packs to packset added
	vec_guard_ratio parameter

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -940,11 +940,16 @@
             node.pack_position = i
 
     def split(self, packlist, vec_reg_size):
+        """ Combination phase creates the biggest packs that are possible.
+            In this step the pack is reduced in size to fit into an
+            vector register.
+        """
         pack = self
         pack_type = self.pack_type()
         max_count = vec_reg_size // pack_type.getsize()
         assert max_count * pack_type.getsize() == vec_reg_size
         while pack.pack_byte_size() > vec_reg_size:
+            assert max_count > 0
             newpack = pack.clone()
             oplist = pack.operations[:max_count]
             newpack.operations = pack.operations[max_count:]
@@ -952,6 +957,7 @@
             pack.update_pack_of_nodes()
             newpack.update_pack_of_nodes()
             pack = newpack
+            packlist.append(newpack)
 
     def rightmost_match_leftmost(self, other):
         """ Check if pack A can be combined with pack B """
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -97,6 +97,8 @@
 
     resop_count = 0 # the count of operations minus debug_merge_points
     vector_instr = 0
+    guard_count = 0
+    blacklist = (rop.CALL, rop.CALL_ASSEMBLER)
     at_least_one_array_access = True
     for i,op in enumerate(loop.operations):
         if op.getopnum() == rop.DEBUG_MERGE_POINT:
@@ -110,6 +112,13 @@
         if op.is_primitive_array_access():
             at_least_one_array_access = True
 
+        if warmstate.vec_ratio > 0.0:
+            if op.getopnum() in blacklist:
+                return True
+
+        if op.is_guard():
+            guard_count += 1
+
     if not at_least_one_array_access:
         return True
 
@@ -119,6 +128,9 @@
     if (float(vector_instr)/float(resop_count)) < warmstate.vec_ratio:
         return True
 
+    if float(guard_count)/float(resop_count) > warmstate.vec_guard_ratio:
+        return True
+
     return False
 
 def cmp_pack_lt(a,b):
diff --git a/rpython/jit/metainterp/test/test_vectorize.py b/rpython/jit/metainterp/test/test_vectorize.py
--- a/rpython/jit/metainterp/test/test_vectorize.py
+++ b/rpython/jit/metainterp/test/test_vectorize.py
@@ -289,15 +289,32 @@
         res = self.meta_interp(f, [i])
         assert res == f(i)
 
-    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3), (15,44.0,22.2)])
+    @py.test.mark.parametrize('i,v1,v2',[(25,2.5,0.3)])
     def test_list_vectorize(self,i,v1,v2):
         myjitdriver = JitDriver(greens = [],
                                 reds = 'auto')
+        class ListF(object):
+            def __init__(self, size, init=0.0):
+                self.list = [init] * size
+            def __getitem__(self, key):
+                if key < 0:
+                    raise IndexError
+                if key >= len(self.list):
+                    raise IndexError
+                return self.list[key]
+            def __setitem__(self, key, value):
+                if key < 0:
+                    raise IndexError
+                if key >= len(self.list):
+                    raise IndexError
+                self.list[key] = value
+            def append(self, value):
+                self.list.append(value)
         def f(d, v1, v2):
-            a = [v1] * i
-            b = [v2] * i
+            a = ListF(d, v1)
+            b = ListF(d, v2)
             i = 0
-            while i < len(a):
+            while i < d:
                 myjitdriver.jit_merge_point()
                 a[i] = a[i] + b[i]
                 i += 1
diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py
--- a/rpython/jit/metainterp/warmspot.py
+++ b/rpython/jit/metainterp/warmspot.py
@@ -72,7 +72,7 @@
                     loop_longevity=0, retrace_limit=5, function_threshold=4,
                     enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, 
                     max_unroll_recursion=7, vec=0, vec_all=0, vec_cost=0,
-                    vec_length=60, vec_ratio=2, **kwds):
+                    vec_length=60, vec_ratio=2, vec_guard_ratio=3, **kwds):
     from rpython.config.config import ConfigError
     translator = interp.typer.annotator.translator
     try:
@@ -100,6 +100,7 @@
         jd.warmstate.set_param_vec_cost(vec_cost)
         jd.warmstate.set_param_vec_length(vec_length)
         jd.warmstate.set_param_vec_ratio(vec_ratio)
+        jd.warmstate.set_param_vec_guard_ratio(vec_guard_ratio)
     warmrunnerdesc.finish()
     if graph_and_interp_only:
         return interp, graph
diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py
--- a/rpython/jit/metainterp/warmstate.py
+++ b/rpython/jit/metainterp/warmstate.py
@@ -315,6 +315,9 @@
     def set_param_vec_ratio(self, value):
         self.vec_ratio = value / 10.0
 
+    def set_param_vec_guard_ratio(self, value):
+        self.vec_guard_ratio = value / 10.0
+
     def disable_noninlinable_function(self, greenkey):
         cell = self.JitCell.ensure_jit_cell_at_key(greenkey)
         cell.flags |= JC_DONT_TRACE_HERE
diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py
--- a/rpython/rlib/jit.py
+++ b/rpython/rlib/jit.py
@@ -559,6 +559,8 @@
     'vec_length': 'the amount of instructions allowed in "all" traces.',
     'vec_ratio': 'an integer (0-10 transfored into a float by X / 10.0) statements that have vector equivalents '
                  'divided by the total number of trace instructions.',
+    'vec_guard_ratio': 'an integer (0-10 transfored into a float by X / 10.0) divided by the'
+                       ' total number of trace instructions.',
 }
 
 PARAMETERS = {'threshold': 1039, # just above 1024, prime
@@ -579,6 +581,7 @@
               'vec_cost': 0,
               'vec_length': 60,
               'vec_ratio': 2,
+              'vec_guard_ratio': 3,
               }
 unroll_parameters = unrolling_iterable(PARAMETERS.items())
 

From noreply at buildbot.pypy.org  Mon Aug 31 10:27:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 10:27:48 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix?
Message-ID: <20150831082748.0608E1C1034@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79311:6a2ee3a1f8da
Date: 2015-08-31 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/6a2ee3a1f8da/

Log:	fix?

diff --git a/rpython/jit/metainterp/test/test_virtual.py b/rpython/jit/metainterp/test/test_virtual.py
--- a/rpython/jit/metainterp/test/test_virtual.py
+++ b/rpython/jit/metainterp/test/test_virtual.py
@@ -919,7 +919,7 @@
         assert res == f(10)
         self.check_aborted_count(0)
         self.check_target_token_count(3)
-        self.check_resops(int_mul=2)
+        self.check_resops(int_mul=3)
 
     def test_nested_loops_bridge(self):
         class Int(object):
@@ -963,7 +963,7 @@
         res = self.meta_interp(f, [32])
         assert res == f(32)
         self.check_aborted_count(0)
-        self.check_target_token_count(3)
+        self.check_target_token_count(4)
 
 
 class VirtualMiscTests:

From noreply at buildbot.pypy.org  Mon Aug 31 10:54:29 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 10:54:29 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: oops,
	don't emit write barriers for ints
Message-ID: <20150831085429.1D1CB1C11D3@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79312:e0a87d87e411
Date: 2015-08-31 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/e0a87d87e411/

Log:	oops, don't emit write barriers for ints

diff --git a/rpython/jit/backend/llsupport/descr.py b/rpython/jit/backend/llsupport/descr.py
--- a/rpython/jit/backend/llsupport/descr.py
+++ b/rpython/jit/backend/llsupport/descr.py
@@ -67,7 +67,7 @@
         assert self.tid != 0
         return self.tid
 
-def get_size_descr(gccache, STRUCT, vtable):
+def get_size_descr(gccache, STRUCT, vtable=lltype.nullptr(rclass.OBJECT_VTABLE)):
     cache = gccache._cache_size
     assert not isinstance(vtable, bool)
     try:
diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -621,7 +621,7 @@
         val = op.getarg(0)
         if not self.write_barrier_applied(val):
             v = op.getarg(1)
-            if (not isinstance(v, ConstPtr) or
+            if (v.type == 'r' and not isinstance(v, ConstPtr) or
                 rgc.needs_write_barrier(v.value)):
                 self.gen_write_barrier(val)
                 #op = op.copy_and_change(rop.SETFIELD_RAW)
@@ -631,7 +631,7 @@
         val = op.getarg(0)
         if not self.write_barrier_applied(val):
             v = op.getarg(2)
-            if (not isinstance(v, ConstPtr) or
+            if (v.type == 'r' and not isinstance(v, ConstPtr) or
                 rgc.needs_write_barrier(v.value)):
                 self.gen_write_barrier_array(val, op.getarg(1))
                 #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py
--- a/rpython/jit/backend/llsupport/test/test_gc.py
+++ b/rpython/jit/backend/llsupport/test/test_gc.py
@@ -23,7 +23,7 @@
     #
     # ---------- gc_malloc ----------
     S = lltype.GcStruct('S', ('x', lltype.Signed))
-    sizedescr = descr.get_size_descr(gc_ll_descr, S, False)
+    sizedescr = descr.get_size_descr(gc_ll_descr, S)
     p = gc_ll_descr.gc_malloc(sizedescr)
     assert record == [(sizedescr.size, p)]
     del record[:]
@@ -143,7 +143,7 @@
 
     def test_gc_malloc(self):
         S = lltype.GcStruct('S', ('x', lltype.Signed))
-        sizedescr = descr.get_size_descr(self.gc_ll_descr, S, False)
+        sizedescr = descr.get_size_descr(self.gc_ll_descr, S)
         p = self.gc_ll_descr.gc_malloc(sizedescr)
         assert lltype.typeOf(p) == llmemory.GCREF
         assert self.llop1.record == [("fixedsize", repr(sizedescr.size),
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -754,7 +754,7 @@
 
     def test_malloc_1(self):
         cpu = self.cpu
-        sizeof = cpu.sizeof(self.S, False)
+        sizeof = cpu.sizeof(self.S, None)
         sizeof.tid = 0
         size = sizeof.size
         loop = self.parse("""
diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py
--- a/rpython/jit/backend/llsupport/test/test_rewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_rewrite.py
@@ -29,13 +29,13 @@
     def check_rewrite(self, frm_operations, to_operations, **namespace):
         S = lltype.GcStruct('S', ('x', lltype.Signed),
                                  ('y', lltype.Signed))
-        sdescr = get_size_descr(self.gc_ll_descr, S, False)
+        sdescr = get_size_descr(self.gc_ll_descr, S)
         sdescr.tid = 1234
         #
         T = lltype.GcStruct('T', ('y', lltype.Signed),
                                  ('z', lltype.Ptr(S)),
                                  ('t', lltype.Signed))
-        tdescr = get_size_descr(self.gc_ll_descr, T, False)
+        tdescr = get_size_descr(self.gc_ll_descr, T)
         tdescr.tid = 5678
         tzdescr = get_field_descr(self.gc_ll_descr, T, 'z')
         #
@@ -55,7 +55,7 @@
         clendescr = cdescr.lendescr
         #
         E = lltype.GcStruct('Empty')
-        edescr = get_size_descr(self.gc_ll_descr, E, False)
+        edescr = get_size_descr(self.gc_ll_descr, E)
         edescr.tid = 9000
         #
         vtable_descr = self.gc_ll_descr.fielddescr_vtable
@@ -1045,13 +1045,13 @@
         self.check_rewrite("""
             []
             p0 = new(descr=tdescr)
-            p1 = getfield_gc(p0, descr=tdescr)
+            p1 = getfield_gc_r(p0, descr=tdescr)
             jump(p1)
         """, """
             []
             p0 = call_malloc_nursery(%(tdescr.size)d)
             setfield_gc(p0, 5678, descr=tiddescr)
             zero_ptr_field(p0, %(tdescr.gc_fielddescrs[0].offset)s)
-            p1 = getfield_gc(p0, descr=tdescr)
+            p1 = getfield_gc_r(p0, descr=tdescr)
             jump(p1)
         """)
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -156,8 +156,8 @@
     def box_for_var(self, res):
         return res
 
-    def create_op(self, opnum, args, res, descr, fail_args):
-        return self.Op(intern(opname[opnum].lower()), args, res, descr, fail_args)
+    def create_op(self, opnum, args, descr, fail_args):
+        return self.Op(intern(opname[opnum].lower()), args, None, descr, fail_args)
 
 
 

From noreply at buildbot.pypy.org  Mon Aug 31 11:08:21 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 11:08:21 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: ups
Message-ID: <20150831090821.0BAF91C1517@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79313:45f08e1b74be
Date: 2015-08-31 11:08 +0200
http://bitbucket.org/pypy/pypy/changeset/45f08e1b74be/

Log:	ups

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -621,8 +621,8 @@
         val = op.getarg(0)
         if not self.write_barrier_applied(val):
             v = op.getarg(1)
-            if (v.type == 'r' and not isinstance(v, ConstPtr) or
-                rgc.needs_write_barrier(v.value)):
+            if (v.type == 'r' and (not isinstance(v, ConstPtr) or
+                rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier(val)
                 #op = op.copy_and_change(rop.SETFIELD_RAW)
         self.emit_op(op)
@@ -631,8 +631,8 @@
         val = op.getarg(0)
         if not self.write_barrier_applied(val):
             v = op.getarg(2)
-            if (v.type == 'r' and not isinstance(v, ConstPtr) or
-                rgc.needs_write_barrier(v.value)):
+            if (v.type == 'r' and (not isinstance(v, ConstPtr) or
+                rgc.needs_write_barrier(v.value))):
                 self.gen_write_barrier_array(val, op.getarg(1))
                 #op = op.copy_and_change(rop.SET{ARRAYITEM,INTERIORFIELD}_RAW)
         self.emit_op(op)

From noreply at buildbot.pypy.org  Mon Aug 31 11:36:30 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 11:36:30 +0200 (CEST)
Subject: [pypy-commit] pypy default: Kill push_gcmap(mov=True): it is always
 followed by taking the gcmap
Message-ID: <20150831093630.1276C1C1C6E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79314:8d8edb9e1343
Date: 2015-08-31 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/8d8edb9e1343/

Log:	Kill push_gcmap(mov=True): it is always followed by taking the gcmap
	back from the stack to store it inside the jitframe. It can be done
	directory with push_gcmap(store=True), which is not more
	instructions (nor a longer one).

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -114,10 +114,7 @@
     def build_frame_realloc_slowpath(self):
         mc = codebuf.MachineCodeBlockWrapper()
         self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats)
-        # this is the gcmap stored by push_gcmap(mov=True) in _check_stack_frame
-        mc.MOV_rs(ecx.value, WORD)
-        gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
-        mc.MOV_br(gcmap_ofs, ecx.value)
+        # the caller already did push_gcmap(store=True)
 
         if IS_X86_64:
             mc.MOV_rs(esi.value, WORD*2)
@@ -147,7 +144,7 @@
             self._load_shadowstack_top_in_ebx(mc, gcrootmap)
             mc.MOV_mr((ebx.value, -WORD), eax.value)
 
-        mc.MOV_bi(gcmap_ofs, 0)
+        self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
         self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats)
         mc.RET()
         self._frame_realloc_slowpath = mc.materialize(self.cpu, [])
@@ -164,6 +161,7 @@
         # the end of this function.
         self._push_all_regs_to_frame(mc, cond_call_register_arguments + [eax],
                                      supports_floats, callee_only)
+        # the caller already did push_gcmap(store=True)
         if IS_X86_64:
             mc.SUB(esp, imm(WORD))     # alignment
             self.set_extra_stack_depth(mc, 2 * WORD)
@@ -182,8 +180,8 @@
             mc.ADD(esp, imm(WORD * 7))
         self.set_extra_stack_depth(mc, 0)
         self._reload_frame_if_necessary(mc, align_stack=True)
+        self.pop_gcmap(mc)   # cancel the push_gcmap(store=True) in the caller
         self._pop_all_regs_from_frame(mc, [], supports_floats, callee_only)
-        self.pop_gcmap(mc)   # push_gcmap(store=True) done by the caller
         mc.RET()
         return mc.materialize(self.cpu, [])
 
@@ -203,10 +201,7 @@
         assert kind in ['fixed', 'str', 'unicode', 'var']
         mc = codebuf.MachineCodeBlockWrapper()
         self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats)
-        # store the gc pattern
-        ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
-        mc.MOV_rs(ecx.value, WORD)
-        mc.MOV_br(ofs, ecx.value)
+        # the caller already did push_gcmap(store=True)
         #
         if kind == 'fixed':
             addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr()
@@ -258,8 +253,7 @@
         self.set_extra_stack_depth(mc, 0)
         self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats)
         mc.MOV(edi, heap(nursery_free_adr))   # load this in EDI
-        # clear the gc pattern
-        mc.MOV_bi(ofs, 0)
+        self.pop_gcmap(mc)   # push_gcmap(store=True) done by the caller
         mc.RET()
         #
         # If the slowpath malloc failed, we raise a MemoryError that
@@ -646,7 +640,7 @@
         jg_location = mc.get_relative_pos()
         mc.MOV_si(WORD, 0xffffff)     # force writing 32 bit
         ofs2 = mc.get_relative_pos() - 4
-        self.push_gcmap(mc, gcmap, mov=True)
+        self.push_gcmap(mc, gcmap, store=True)
         mc.CALL(imm(self._frame_realloc_slowpath))
         # patch the JG above
         offset = mc.get_relative_pos() - jg_location
@@ -1797,12 +1791,9 @@
         self.mc.JMP(imm(target))
         return startpos
 
-    def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False):
+    def push_gcmap(self, mc, gcmap, push=False, store=False):
         if push:
             mc.PUSH(imm(rffi.cast(lltype.Signed, gcmap)))
-        elif mov:
-            mc.MOV(RawEspLoc(0, REF),
-                   imm(rffi.cast(lltype.Signed, gcmap)))
         else:
             assert store
             ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap')
@@ -2280,7 +2271,7 @@
         self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr = self.mc.get_relative_pos()
         # save the gcmap
-        self.push_gcmap(self.mc, gcmap, mov=True)
+        self.push_gcmap(self.mc, gcmap, store=True)
         self.mc.CALL(imm(follow_jump(self.malloc_slowpath)))
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
@@ -2301,7 +2292,7 @@
         self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later
         jmp_adr = self.mc.get_relative_pos()
         # save the gcmap
-        self.push_gcmap(self.mc, gcmap, mov=True)
+        self.push_gcmap(self.mc, gcmap, store=True)
         self.mc.CALL(imm(follow_jump(self.malloc_slowpath)))
         offset = self.mc.get_relative_pos() - jmp_adr
         assert 0 < offset <= 127
@@ -2354,7 +2345,7 @@
         assert 0 < offset <= 127
         self.mc.overwrite(jmp_adr0-1, chr(offset))
         # save the gcmap
-        self.push_gcmap(self.mc, gcmap, mov=True)   # mov into RawEspLoc(0)
+        self.push_gcmap(self.mc, gcmap, store=True)
         if kind == rewrite.FLAG_ARRAY:
             self.mc.MOV_si(WORD, itemsize)
             self.mc.MOV(edi, lengthloc)

From noreply at buildbot.pypy.org  Mon Aug 31 11:36:32 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 11:36:32 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <20150831093632.430801C1C6E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79315:37723950bd21
Date: 2015-08-31 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/37723950bd21/

Log:	merge heads

diff --git a/lib_pypy/greenlet.egg-info b/lib_pypy/greenlet.egg-info
--- a/lib_pypy/greenlet.egg-info
+++ b/lib_pypy/greenlet.egg-info
@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: greenlet
-Version: 0.4.7
+Version: 0.4.9
 Summary: Lightweight in-process concurrent programming
 Home-page: https://github.com/python-greenlet/greenlet
 Author: Ralf Schmitt (for CPython), PyPy team
diff --git a/lib_pypy/greenlet.py b/lib_pypy/greenlet.py
--- a/lib_pypy/greenlet.py
+++ b/lib_pypy/greenlet.py
@@ -1,7 +1,7 @@
 import sys
 import _continuation
 
-__version__ = "0.4.7"
+__version__ = "0.4.9"
 
 # ____________________________________________________________
 # Exceptions

From noreply at buildbot.pypy.org  Mon Aug 31 11:53:17 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 11:53:17 +0200 (CEST)
Subject: [pypy-commit] pypy default: For pypy translated without the _thread
 module: optimization
Message-ID: <20150831095317.C0DA61C01F2@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79316:9458d6ef6c53
Date: 2015-08-31 11:53 +0200
http://bitbucket.org/pypy/pypy/changeset/9458d6ef6c53/

Log:	For pypy translated without the _thread module: optimization

diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -9,6 +9,7 @@
     implementation for this feature, and patches 'space.threadlocals' when
     'thread' is initialized.
     """
+    _immutable_fields_ = ['_value?']
     _value = None
 
     def get_ec(self):

From noreply at buildbot.pypy.org  Mon Aug 31 13:16:34 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 13:16:34 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: we forgot to put enter/leave
 portal frame into a does not have effect on heap
Message-ID: <20150831111634.2EC431C0726@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79317:b63169a0ed5e
Date: 2015-08-31 13:16 +0200
http://bitbucket.org/pypy/pypy/changeset/b63169a0ed5e/

Log:	we forgot to put enter/leave portal frame into a does not have
	effect on heap

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -309,6 +309,8 @@
             opnum == rop.UNICODESETITEM or       # no effect on GC struct/array
             opnum == rop.DEBUG_MERGE_POINT or    # no effect whatsoever
             opnum == rop.JIT_DEBUG or            # no effect whatsoever
+            opnum == rop.ENTER_PORTAL_FRAME or   # no effect whatsoever
+            opnum == rop.LEAVE_PORTAL_FRAME or   # no effect whatsoever
             opnum == rop.COPYSTRCONTENT or       # no effect on GC struct/array
             opnum == rop.COPYUNICODECONTENT):    # no effect on GC struct/array
             return

From noreply at buildbot.pypy.org  Mon Aug 31 13:56:38 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 13:56:38 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: changes to make parsing work
Message-ID: <20150831115638.F270C1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79318:ff5f9e91ac2a
Date: 2015-08-31 13:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ff5f9e91ac2a/

Log:	changes to make parsing work

diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -293,7 +293,7 @@
 
         return opnum, args, descr, fail_args
 
-    def create_op(self, opnum, args, descr, fail_args):
+    def create_op(self, opnum, args, res, descr, fail_args):
         if opnum in ALL_ESCAPE_OPS:
             op = ALL_ESCAPE_OPS[opnum]()
             op.initarglist(args)
@@ -319,13 +319,13 @@
         opnum, args, descr, fail_args = self.parse_op(op)
         if res in self.vars:
             raise ParseError("Double assign to var %s in line: %s" % (res, line))
-        resop = self.create_op(opnum, args, descr, fail_args)
+        resop = self.create_op(opnum, args, res, descr, fail_args)
         self.vars[res] = resop
         return resop
 
     def parse_op_no_result(self, line):
         opnum, args, descr, fail_args = self.parse_op(line)
-        res = self.create_op(opnum, args, descr, fail_args)
+        res = self.create_op_no_result(opnum, args, descr, fail_args)
         return res
 
     def parse_next_op(self, line):
diff --git a/rpython/tool/jitlogparser/parser.py b/rpython/tool/jitlogparser/parser.py
--- a/rpython/tool/jitlogparser/parser.py
+++ b/rpython/tool/jitlogparser/parser.py
@@ -156,8 +156,13 @@
     def box_for_var(self, res):
         return res
 
-    def create_op(self, opnum, args, descr, fail_args):
-        return self.Op(intern(opname[opnum].lower()), args, None, descr, fail_args)
+    def create_op(self, opnum, args, res, descr, fail_args):
+        return self.Op(intern(opname[opnum].lower()), args, res,
+                       descr, fail_args)
+
+    def create_op_no_result(self, opnum, args, descr, fail_args):
+        return self.Op(intern(opname[opnum].lower()), args, None,
+                       descr, fail_args)
 
 
 

From noreply at buildbot.pypy.org  Mon Aug 31 14:34:48 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 14:34:48 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: One more benchmark
Message-ID: <20150831123448.A65111C0726@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: 
Changeset: r333:7b876ef4812e
Date: 2015-08-31 14:35 +0200
http://bitbucket.org/pypy/benchmarks/changeset/7b876ef4812e/

Log:	One more benchmark

diff --git a/warmup/bridges.py b/warmup/bridges.py
new file mode 100644
--- /dev/null
+++ b/warmup/bridges.py
@@ -0,0 +1,29 @@
+
+for i in range(100):
+    exec """
+def f():
+    s = 0
+    i = 0
+    while i < 10000:
+        i += 1
+        if i % 2 == 0:
+            s += 1
+        if i % 3 == 0:
+            s += 1
+        if i % 5 == 0:
+            s += 1
+        if i % 7 == 0:
+            s += 1
+        if i % 11 == 0:
+            s += 1
+        if i % 13 == 0:
+            s += 1
+        if i % 17 == 0:
+            s += 1
+        if i % 19 == 0:
+            s += 1
+        if i % 23 == 0:
+            s += 1
+
+f()
+"""

From noreply at buildbot.pypy.org  Mon Aug 31 15:06:07 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 31 Aug 2015 15:06:07 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: resolving some issues introduced
 by the simpler combination and separate splitting phase, 
Message-ID: <20150831130607.6B46F1C1034@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79319:75f6354522df
Date: 2015-08-31 14:33 +0200
http://bitbucket.org/pypy/pypy/changeset/75f6354522df/

Log:	resolving some issues introduced by the simpler combination and
	separate splitting phase, needs to remove packs that are not full

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -240,6 +240,11 @@
     def getcount(self):
         return self.count
 
+    def pack_byte_size(self, pack):
+        if len(pack.operations) == 0:
+            return 0
+        return self.getsize() * pack.opcount()
+
 
 PT_GENERIC = PackType(PackType.UNKNOWN_TYPE, -1, False)
 PT_FLOAT_2 = PackType(FLOAT, 4, False, 2)
@@ -873,11 +878,43 @@
         assert not isinstance(box, BoxVector)
         self.box_to_vbox[box] = (off, vector)
 
+def opcount_filling_vector_register(pack, vec_reg_size):
+    """ how many operations of that kind can one execute
+        with a machine instruction of register size X?
+    """
+    pack_type = pack.input_type
+    if pack_type is None:
+        pack_type = pack.output_type # load operations
+
+    op = pack.leftmost()
+    if op.casts_box():
+        count = pack_type.getcount()
+        return count
+
+    count = vec_reg_size // pack_type.getsize()
+    return count
+
+def maximum_byte_size(pack, vec_reg_size):
+    """ The maxmum size in bytes the operation is able to
+        process with the hardware register and the operation
+        semantics.
+    """
+    op = pack.leftmost()
+    if op.casts_box():
+        # casting is special, often only takes a half full vector
+        pack_type = pack.input_type
+        if pack_type is None:
+            pack_type = self.output_type # load operations
+        return pack_type.byte_size()
+    return vec_reg_size
+
 class Pack(object):
     """ A pack is a set of n statements that are:
         * isomorphic
         * independent
     """
+    FULL = 0
+
     def __init__(self, ops, input_type, output_type):
         self.operations = ops
         self.accum = None
@@ -899,30 +936,43 @@
             ptype = self.output_type
         return ptype
 
-    def pack_byte_size(self):
-        return self.pack_type().getsize() * self.opcount()
+    def input_byte_size(self):
+        """ The amount of bytes the operations need with the current
+            entries in self.operations. E.g. cast_singlefloat_to_float
+            takes only #2 operations.
+        """
+        return self._byte_size(self.input_type)
+
+    def output_byte_size(self):
+        """ The amount of bytes the operations need with the current
+            entries in self.operations. E.g. vec_load(..., descr=short) 
+            with 10 operations returns 20
+        """
+        return self._byte_size(self.output_type)
+
+    def pack_load(self, vec_reg_size):
+        """ Returns the load of the pack. A value
+            smaller than 0 indicates that it is empty
+            or nearly empty, zero indicates that all slots
+            are used and > 0 indicates that too many operations
+            are in this pack instance.
+        """
+        if len(self.operations) == 0:
+            return -1
+        size = maximum_byte_size(self, vec_reg_size)
+        if self.input_type is None:
+            # e.g. load operations
+            return self.output_type.pack_byte_size(self) - size
+        # default only consider the input type
+        # e.g. store operations, int_add, ...
+        return self.input_type.pack_byte_size(self) - size
+
 
     def is_full(self, vec_reg_size):
         """ If one input element times the opcount is equal
             to the vector register size, we are full!
         """
-        ptype = self.pack_type()
-        op = self.leftmost()
-        if op.casts_box():
-            cur_bytes = ptype.getsize() * self.opcount()
-            max_bytes = self.input_type.byte_size()
-            assert cur_bytes <= max_bytes
-            return cur_bytes == max_bytes
-
-        bytes = self.pack_byte_size()
-        assert bytes <= vec_reg_size
-        if bytes == vec_reg_size:
-            return True
-        if ptype.getcount() != -1:
-            size = ptype.getcount() * ptype.getsize()
-            assert bytes <= size
-            return bytes == size
-        return False
+        return self.pack_load(vec_reg_size) == Pack.FULL
 
     def opnum(self):
         assert len(self.operations) > 0
@@ -930,9 +980,8 @@
 
     def clear(self):
         for node in self.operations:
-            if node.pack is not self:
-                node.pack = None
-                node.pack_position = -1
+            node.pack = None
+            node.pack_position = -1
 
     def update_pack_of_nodes(self):
         for i,node in enumerate(self.operations):
@@ -945,19 +994,28 @@
             vector register.
         """
         pack = self
-        pack_type = self.pack_type()
-        max_count = vec_reg_size // pack_type.getsize()
-        assert max_count * pack_type.getsize() == vec_reg_size
-        while pack.pack_byte_size() > vec_reg_size:
-            assert max_count > 0
-            newpack = pack.clone()
-            oplist = pack.operations[:max_count]
-            newpack.operations = pack.operations[max_count:]
+        while pack.pack_load(vec_reg_size) > Pack.FULL:
+            pack.clear()
+            oplist, newoplist = pack.slice_operations(vec_reg_size)
             pack.operations = oplist
             pack.update_pack_of_nodes()
-            newpack.update_pack_of_nodes()
-            pack = newpack
-            packlist.append(newpack)
+            assert pack.is_full(vec_reg_size)
+            #
+            newpack = pack.clone(newoplist)
+            load = newpack.pack_load(vec_reg_size)
+            if load >= Pack.FULL:
+                pack = newpack
+                packlist.append(newpack)
+            else:
+                newpack.clear()
+
+    def slice_operations(self, vec_reg_size):
+        count = opcount_filling_vector_register(self, vec_reg_size)
+        newoplist = self.operations[count:]
+        oplist = self.operations[:count]
+        assert len(newoplist) + len(oplist) == len(self.operations)
+        assert len(newoplist) != 0
+        return oplist, newoplist
 
     def rightmost_match_leftmost(self, other):
         """ Check if pack A can be combined with pack B """
@@ -974,14 +1032,16 @@
         return rightmost is leftmost and accum
 
     def __repr__(self):
+        if len(self.operations) == 0:
+            return "Pack(-, [])"
         opname = self.operations[0].getoperation().getopname()
         return "Pack(%s,%r)" % (opname, self.operations)
 
     def is_accumulating(self):
         return self.accum is not None
 
-    def clone(self):
-        cloned = Pack(self.operations, self.input_type, self.output_type)
+    def clone(self, oplist):
+        cloned = Pack(oplist, self.input_type, self.output_type)
         cloned.accum = self.accum
         return cloned
 
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -452,3 +452,19 @@
         v9[i64|2] = vec_int_and(v4[i64|2], v8[i64|2])
         """, False)
         self.assert_equal(loop2, loop3)
+
+    def test_split_cast(self):
+        trace = self.parse("""
+        f10 = cast_int_to_float(i1)
+        f11 = cast_int_to_float(i2)
+        f12 = cast_int_to_float(i3)
+        f13 = cast_int_to_float(i4)
+        """)
+        pack = self.pack(trace, 0, 4, I64, F32)
+        packs = []
+        pack.split(packs, 16)
+        packs.append(pack)
+        assert len(packs) == 2
+
+
+
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_vectorize.py
@@ -1427,5 +1427,51 @@
         opt = self.schedule(self.parse_loop(trace))
         self.debug_print_operations(opt.loop)
 
+    def test_arraylen(self):
+        trace = """
+        [i45, i33, p40]
+        # while i < len(l):
+        # LOAD_FAST i
+        # LOAD_GLOBAL len
+        guard_not_invalidated(descr=) [i33,p40]
+        # LOAD_FAST l
+        # CALL_FUNCTION 1
+        # COMPARE_OP <
+        i50 = int_lt(i45, i33)
+        guard_true(i50) [i50,i33,p40]
+        # POP_JUMP_IF_FALSE 70
+        # l[i] = l[i] + 1
+        # LOAD_FAST l
+        # LOAD_FAST i
+        # BINARY_SUBSCR 
+        i51 = uint_ge(i45, i33)
+        guard_false(i51) [i50, i45]
+        i52 = getarrayitem_gc(p40, i45, descr=intarraydescr)
+        # LOAD_CONST 1
+        # BINARY_ADD 
+        i53 = int_add(i52, 1)
+        #guard_no_overflow(descr=) []
+        # LOAD_FAST l
+        # LOAD_FAST i
+        # STORE_SUBSCR 
+        setarrayitem_gc(p40, i45, i53, descr=intarraydescr)
+        # i += 1
+        # LOAD_FAST i
+        # LOAD_CONST 1
+        # INPLACE_ADD 
+        i54 = int_add(i45,1)
+        # STORE_FAST i
+        # JUMP_ABSOLUTE 21
+        #getfield_raw_i(140199654614400, descr=)
+        #None = i55 < 0
+        #guard(i56 is false)
+        # LOAD_FAST i
+        #i34 = arraylen_gc(p40, descr=)
+        jump(i54, i33, p40)
+        """
+        opt = self.vectorize(self.parse_loop(trace))
+        self.debug_print_operations(opt.loop)
+
+
 class TestLLtype(BaseTestVectorize, LLtypeMixin):
     pass
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -133,10 +133,6 @@
 
     return False
 
-def cmp_pack_lt(a,b):
-    return a.left.getindex() < b.left.getindex()
-packsort = listsort.make_timsort_class(lt=cmp_pack_lt)
-
 class VectorizingOptimizer(Optimizer):
     """ Try to unroll the loop and find instructions to group """
 
@@ -401,13 +397,6 @@
         """
         if len(self.packset.packs) == 0:
             raise NotAVectorizeableLoop()
-        #packsort(self.packset.packs).sort()
-        #if not we_are_translated():
-        #    # ensure we are really sorted!
-        #    x = 0
-        #    for i,pack in enumerate(self.packset.packs):
-        #        assert x <= pack.left.getindex()
-        #        x = pack.left.getindex()
         i = 0
         j = 0
         end_ij = len(self.packset.packs)
@@ -423,33 +412,6 @@
                         continue
                     pack1 = self.packset.packs[i]
                     pack2 = self.packset.packs[j]
-                    # remove intermediate
-                    left = pack1.operations[0]
-                    #if left in orphan:
-                    #    # a pack was filled, thus the rhs was put
-                    #    # into the orphan map.
-                    #    if orphan[left] is False:
-                    #        # this pack might be redundant if pack1.right
-                    #        # is the at the left position in another pack
-                    #        assert pack1.opcount() == 2
-                    #        right = pack1.operations[1]
-                    #        orphan[right] = True
-                    #        pack1.clear()
-                    #        del self.packset.packs[i]
-                    #        end_ij -= 1
-                    #        continue
-                    #    else:
-                    #        # left is not an orphan, this pack proves that
-                    #        # there might be more packs
-                    #        del orphan[left]
-                    # check if the pack is already full
-                    #if pack1.is_full(self.cpu.vector_register_size):
-                    #    right = pack1.operations[-1]
-                    #    # False indicates that the next pair might not
-                    #    # be needed, because left is already computed
-                    #    # in another set
-                    #    orphan[right] = False
-                    #    break
                     if pack1.rightmost_match_leftmost(pack2):
                         end_ij = self.packset.combine(i,j)
                     else:
@@ -460,11 +422,14 @@
                 j = 0
             if len_before == len(self.packset.packs):
                 break
+        newpacks = []
+        vec_reg_size = self.cpu.vector_register_size
         for pack in self.packset.packs:
-            if pack.pack_byte_size() > self.cpu.vector_register_size:
-                pack.split(self.packset.packs, self.cpu.vector_register_size)
-            else:
-                pack.update_pack_of_nodes()
+            if pack.pack_load(vec_reg_size) > Pack.FULL:
+                pack.split(newpacks, vec_reg_size)
+                continue
+            pack.update_pack_of_nodes()
+        self.packset.packs.extend(newpacks)
 
         if not we_are_translated():
             # some test cases check the accumulation variables
@@ -483,9 +448,10 @@
                 if accum:
                     self.packset.accum_vars[accum.var] = accum.pos
 
-                print " %dx %s (accum? %d) " % (len(pack.operations),
-                                                pack.operations[0].op.getopname(),
-                                                accum is not None)
+                print " %dx %s " % (len(pack.operations),
+                                    pack.operations[0].op.getopname())
+                if accum:
+                    print "   accumulates!"
             if fail:
                 assert False
 

From noreply at buildbot.pypy.org  Mon Aug 31 15:06:09 2015
From: noreply at buildbot.pypy.org (plan_rich)
Date: Mon, 31 Aug 2015 15:06:09 +0200 (CEST)
Subject: [pypy-commit] pypy vecopt-merge: fixed the scheduling tests,
 two of them where wrong and execute vector instructions on half
 filled vector registers
Message-ID: <20150831130609.7E5B91C1034@cobra.cs.uni-duesseldorf.de>

Author: Richard Plangger 
Branch: vecopt-merge
Changeset: r79320:5a31e0903f8d
Date: 2015-08-31 15:06 +0200
http://bitbucket.org/pypy/pypy/changeset/5a31e0903f8d/

Log:	fixed the scheduling tests, two of them where wrong and execute
	vector instructions on half filled vector registers

diff --git a/rpython/jit/metainterp/optimizeopt/schedule.py b/rpython/jit/metainterp/optimizeopt/schedule.py
--- a/rpython/jit/metainterp/optimizeopt/schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/schedule.py
@@ -272,21 +272,12 @@
         self.sched_data = sched_data
         self.vecops = oplist
         self.costmodel = sched_data.costmodel
-        #
         self.input_type = pack.input_type
         self.output_type = pack.output_type
         #
         self.check_if_pack_supported(pack)
-
-        #
-        if self.must_be_full_but_is_not(pack):
-            for op in pack.operations:
-                operation = op.getoperation()
-                self.sched_data.unpack_from_vector(operation, scheduler)
-                self.vecops.append(operation)
-        else:
-            self.pack = pack
-            self.transform_pack()
+        self.pack = pack
+        self.transform_pack()
         #
         self.pack = None
         self.costmodel = None
@@ -295,9 +286,6 @@
         self.input_type = None
         self.output_type = None
 
-    def must_be_full_but_is_not(self, pack):
-        return False
-
     def before_argument_transform(self, args):
         pass
 
@@ -1008,6 +996,8 @@
                 packlist.append(newpack)
             else:
                 newpack.clear()
+                newpack.operations = []
+                break
 
     def slice_operations(self, vec_reg_size):
         count = opcount_filling_vector_register(self, vec_reg_size)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_schedule.py
@@ -24,8 +24,9 @@
 I16 = PackType('i',2,True,8)
 
 class FakePackSet(PackSet):
-    def __init__(self):
-        self.packs = None
+    def __init__(self, packs):
+        self.packs = packs
+        self.vec_reg_size = 16
 
 class FakeDependencyGraph(DependencyGraph):
     """ A dependency graph that is able to emit every instruction
@@ -115,8 +116,7 @@
                 pair = Pair(o1,o2,pack.input_type,pack.output_type)
                 pairs.append(pair)
 
-        opt.packset = FakePackSet()
-        opt.packset.packs = pairs
+        opt.packset = FakePackSet(pairs)
 
         if not prepend_invariant:
             def pio(oplist, labels):
@@ -149,7 +149,8 @@
         loop2 = self.schedule(loop1, [pack1])
         loop3 = self.parse("""
         v10[i32|4] = vec_raw_load(p0, i0, 4, descr=float)
-        v11[i32|2] = vec_raw_load(p0, i4, 2, descr=float)
+        f10 = raw_load(p0, i4, descr=float)
+        f11 = raw_load(p0, i5, descr=float)
         """, False)
         self.assert_equal(loop2, loop3)
 
@@ -379,17 +380,19 @@
         loop1 = self.parse("""
         i10 = raw_load(p0, i1, descr=float)
         i11 = raw_load(p0, i2, descr=float)
+        i12 = raw_load(p0, i3, descr=float)
+        i13 = raw_load(p0, i4, descr=float)
         raw_store(p0, i3, i10, descr=float)
         raw_store(p0, i4, i11, descr=float)
         """)
-        pack1 = self.pack(loop1, 0, 2, None, I32_2)
-        pack2 = self.pack(loop1, 2, 4, I32_2, None)
+        pack1 = self.pack(loop1, 0, 4, None, I32)
+        pack2 = self.pack(loop1, 4, 6, I32_2, None)
         loop2 = self.schedule(loop1, [pack1,pack2], prepend_invariant=True)
         loop3 = self.parse("""
-        v1[i32|2] = vec_raw_load(p0, i1, 2, descr=float)
-        i10 = vec_int_unpack(v1[i32|2], 0, 1)
+        v1[i32|4] = vec_raw_load(p0, i1, 4, descr=float)
+        i10 = vec_int_unpack(v1[i32|4], 0, 1)
         raw_store(p0, i3, i10, descr=float)
-        i11 = vec_int_unpack(v1[i32|2], 1, 1)
+        i11 = vec_int_unpack(v1[i32|4], 1, 1)
         raw_store(p0, i4, i11, descr=float)
         """, False)
         # unfortunate ui32 is the type for float32... the unsigned u is for
@@ -466,5 +469,13 @@
         packs.append(pack)
         assert len(packs) == 2
 
+    def test_combine_packset_nearly_empty_pack(self):
+        trace = self.parse("""
+        i10 = int_add(i1, i3)
+        i11 = int_add(i2, i3)
+        """)
+        pack = self.pack(trace, 0, 2, I16, I16)
+        packset = FakePackSet([pack])
+        packset.split_overloaded_packs()
+        assert len(packset.packs) == 0
 
-
diff --git a/rpython/jit/metainterp/optimizeopt/vectorize.py b/rpython/jit/metainterp/optimizeopt/vectorize.py
--- a/rpython/jit/metainterp/optimizeopt/vectorize.py
+++ b/rpython/jit/metainterp/optimizeopt/vectorize.py
@@ -317,9 +317,7 @@
         loop = self.loop
         operations = loop.operations
 
-        self.packset = PackSet(self.dependency_graph, operations,
-                               self.unroll_count, self.smallest_type_bytes,
-                               self.cpu)
+        self.packset = PackSet(self.cpu.vector_register_size)
         graph = self.dependency_graph
         memory_refs = graph.memory_refs.items()
         # initialize the pack set
@@ -422,14 +420,8 @@
                 j = 0
             if len_before == len(self.packset.packs):
                 break
-        newpacks = []
-        vec_reg_size = self.cpu.vector_register_size
-        for pack in self.packset.packs:
-            if pack.pack_load(vec_reg_size) > Pack.FULL:
-                pack.split(newpacks, vec_reg_size)
-                continue
-            pack.update_pack_of_nodes()
-        self.packset.packs.extend(newpacks)
+
+        self.packset.split_overloaded_packs()
 
         if not we_are_translated():
             # some test cases check the accumulation variables
@@ -700,15 +692,10 @@
     return False
 
 class PackSet(object):
-    def __init__(self, dependency_graph, operations, unroll_count,
-                 smallest_type_bytes, cpu):
+    _attrs_ = ('packs', 'vec_reg_size')
+    def __init__(self, vec_reg_size):
         self.packs = []
-        self.dependency_graph = dependency_graph
-        self.operations = operations
-        self.unroll_count = unroll_count
-        self.smallest_type_bytes = smallest_type_bytes
-        self.cpu = cpu
-        self.vec_reg_size = self.cpu.vector_register_size
+        self.vec_reg_size = vec_reg_size
 
     def pack_count(self):
         return len(self.packs)
@@ -898,3 +885,17 @@
             sched_data.setvector_of_box(accum.getoriginalbox(), 0, result) # prevent it from expansion
             renamer.start_renaming(accum.getoriginalbox(), result)
 
+    def split_overloaded_packs(self):
+        newpacks = []
+        for i,pack in enumerate(self.packs):
+            load = pack.pack_load(self.vec_reg_size)
+            if load > Pack.FULL:
+                pack.split(newpacks, self.vec_reg_size)
+                continue
+            if load < Pack.FULL:
+                pack.clear()
+                self.packs[i] = None
+                continue
+            pack.update_pack_of_nodes()
+        self.packs = [pack for pack in self.packs + newpacks if pack]
+

From noreply at buildbot.pypy.org  Mon Aug 31 17:27:43 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 31 Aug 2015 17:27:43 +0200 (CEST)
Subject: [pypy-commit] pypy.org extradoc: update downloads for 2.6.1
Message-ID: <20150831152743.563481C1517@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: extradoc
Changeset: r631:43c668d35ecc
Date: 2015-08-30 22:46 +0300
http://bitbucket.org/pypy/pypy.org/changeset/43c668d35ecc/

Log:	update downloads for 2.6.1

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -74,7 +74,7 @@
 performance improvements.
    
 
    We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:
    
 
@@ -113,20 +113,35 @@
 degrees of being up-to-date.
    •
    • -
    -

    Python2.7 compatible PyPy 2.6.0

    -
      -
    • Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04) (see [1] below)
      • -
    • Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04) (see [1] below)
      • -
    • ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian) (see [1] below)
      • -
    • ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring) (see [1] below)
      • -
    • ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2, Ubuntu Precise) (see [1] below)
      • -
    • Mac OS/X binary (64bit)
      • -
    • Windows binary (32bit) (you might need the VS 2008 runtime library -installer vcredist_x86.exe.)
      • -
    • Source (tar.bz2); Source (zip). See below for more about the sources.
      • -
    • All our downloads, including previous versions. We also have a -mirror, but please use only if you have troubles accessing the links above
      • +
      +

      Python2.7 compatible PyPy 2.6.1

      +
      @@ -190,7 +205,7 @@ uncompressed, they run in-place. For now you can uncompress them either somewhere in your home directory or, say, in /opt, and if you want, put a symlink from somewhere like -/usr/local/bin/pypy to /path/to/pypy-2.6.0/bin/pypy. Do +/usr/local/bin/pypy to /path/to/pypy-2.6.1/bin/pypy. Do not move or copy the executable pypy outside the tree – put a symlink to it, otherwise it will not find its libraries.

      @@ -232,7 +247,7 @@

    • Get the source code. The following packages contain the source at the same revision as the above binaries:

      Or you can checkout the current trunk using Mercurial (the trunk usually works and is of course more up-to-date):

      @@ -340,17 +355,18 @@

      Checksums

      Here are the checksums for each of the downloads

      -

      pypy-2.6.0 md5:

      +

      pypy-2.6.1 md5:

      -7ea431ab25737462e23a65b9c3819de3  pypy-2.6.0-linux64.tar.bz2
-edec421b668d945c3922bb6a543b58c8  pypy-2.6.0-linux-armel.tar.bz2
-884f1e49fb130c0b8216795d7017025a  pypy-2.6.0-linux-armhf-raring.tar.bz2
-f6e7a7ffc775150994319c2d6f932c41  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
-802f5122a691718dddcece43687cb2cf  pypy-2.6.0-linux.tar.bz2
-63d49e5ead794e6a83c9d87b577d806d  pypy-2.6.0-osx64.tar.bz2
-b09ab96f50ab3021d020e321f210e4c0  pypy-2.6.0-src.tar.bz2
-81297e691d861adb0c89e8a94ef44e8b  pypy-2.6.0-src.zip
-6a1e5451e98a19027333368280b465e1  pypy-2.6.0-win32.zip
+2346426786459fdc72ad03fe75a98b35  pypy-2.6.1-freebsd64.tar.bz2
+eb265bad9f61029f7a6bc5032d0e5459  pypy-2.6.1-linux64.tar.bz2
+45418996d8d81c7d72437d4a6e610fb3  pypy-2.6.1-linux-armel.tar.bz2
+980cce0274b0a80d8b2da1242ab323e9  pypy-2.6.1-linux-armhf-raring.tar.bz2
+56e80fdb9b3cdec59b4f38af2456c63c  pypy-2.6.1-linux-armhf-raspbian.tar.bz2
+36c6d0ea043027e49cabb6a31fb3388a  pypy-2.6.1-linux.tar.bz2
+d6f847a3c2fb795f5f4fbd670459908c  pypy-2.6.1-osx64.tar.bz2
+7e53f72eeb6d9947fd5db6872213404d  pypy-2.6.1-src.tar.bz2
+7df9dce6c6d353069463e4ecdf460fbf  pypy-2.6.1-src.zip
+890465185948f4043c7104c05bf75fe2  pypy-2.6.1-win32.zip

      pypy3-2.4.0 md5:

      @@ -369,18 +385,19 @@
 2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
 009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
      -

      pypy-2.6.0 sha1:

      -
      -03374692eac05b5402b8fb16be9284efda5a0440  pypy-2.6.0-linux64.tar.bz2
-a3029480d3da45793b4a754ef86fca76f5aa0664  pypy-2.6.0-linux-armel.tar.bz2
-075864a8a8148c9439f8f1b59690d946d7c44ce8  pypy-2.6.0-linux-armhf-raring.tar.bz2
-0fa9a25781659e2b1d40482af315f5b7e73d7473  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
-792db4424bf1654ee66f6dc7bdddc89746cef3f4  pypy-2.6.0-linux.tar.bz2
-fb1da056f355a19181b1a4e13699119a92807ccc  pypy-2.6.0-osx64.tar.bz2
-d6580ed01c0b963ef3735e810bc750b8d44f11f2  pypy-2.6.0-src.tar.bz2
-b768f87e8db6432bff3970bbb7a664c412fb4e1c  pypy-2.6.0-src.zip
-a4e212172f9656021d69af4baddc24f7139dde8c  pypy-2.6.0-win32.zip
-
      +
      +
      pypy-2.6.1 sha1::
      +
      119148e67e94419e86ba11b6cfab826c093496eb pypy-2.6.1-freebsd64.tar.bz2 +20197f670edb3783565bd092c14658fca61c695a pypy-2.6.1-linux64.tar.bz2 +033f65def368025f5e051320be233ec60102f143 pypy-2.6.1-linux-armel.tar.bz2 +672bc22ad81c471b0d8622e826bf16c522bfbeb0 pypy-2.6.1-linux-armhf-raring.tar.bz2 +6c2b1113237da87867b0b06a044b26f506050abc pypy-2.6.1-linux-armhf-raspbian.tar.bz2 +1f27ed11398172a45f870cc37cfd0992bf49fba8 pypy-2.6.1-linux.tar.bz2 +a7b2dd8380ae96a9a8934e99d898853257c2e7e4 pypy-2.6.1-osx64.tar.bz2 +bf0f986bc64b71489983a12f2eb9b504d2ac6fd4 pypy-2.6.1-src.tar.bz2 +d4f7e6b7a2e85ea10365be5cadf46bc5d618dab3 pypy-2.6.1-src.zip +38f710c16f06cc4b99ff2b5bda902624711149bb pypy-2.6.1-win32.zip
      +

      pypy3-2.4.0 sha1:

       7d715742f6929351b310a2ca3b924cab35913089  pypy3-2.4.0-linux64.tar.bz2
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -14,12 +14,12 @@
 
 We provide binaries for x86 and ARM Linux, Mac OS/X and Windows for:
 
-* the Python2.7 compatible release — **PyPy 2.6.0** — (`what's new in PyPy 2.6.0?`_)
+* the Python2.7 compatible release — **PyPy 2.6.1** — (`what's new in PyPy 2.6.1?`_)
 * the Python3.2.5 compatible release — **PyPy3 2.4.0** — (`what's new in PyPy3 2.4.0?`_).
 
 * the Python2.7 Software Transactional Memory special release — **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy 2.6.0?: http://doc.pypy.org/en/latest/release-2.6.0.html
+.. _what's new in PyPy 2.6.1?: http://doc.pypy.org/en/latest/release-2.6.1.html
 .. _what's new in PyPy3 2.4.0?: http://doc.pypy.org/en/latest/release-pypy3-2.4.0.html
 
 
@@ -73,7 +73,7 @@
 .. _`portable Linux binaries`: https://github.com/squeaky-pl/portable-pypy#portable-pypy-distribution-for-linux
 
 
-Python2.7 compatible PyPy 2.6.0
+Python2.7 compatible PyPy 2.6.1
 -----------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -82,21 +82,23 @@
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below)
 * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)`__ (see ``[1]`` below)
 * `Mac OS/X binary (64bit)`__
+* `FreeBSD 9.2 x86 64 bit`__ (see ``[1]` below)
 * `Windows binary (32bit)`__ (you might need the VS 2008 runtime library
   installer `vcredist_x86.exe`_.)
 * `Source (tar.bz2)`__; `Source (zip)`__.  See below for more about the sources.
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-linux.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-freebsd64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://cobra.cs.uni-duesseldorf.de/~buildmaster/mirror/
@@ -190,7 +192,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy-2.6.0/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy-2.6.1/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -246,9 +248,9 @@
 1. Get the source code.  The following packages contain the source at
    the same revision as the above binaries:
 
-   * `pypy-2.6.0-src.tar.bz2`__ (sources)
+   * `pypy-2.6.1-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.0-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-2.6.1-src.tar.bz2
 
    Or you can checkout the current trunk using Mercurial_ (the trunk
    usually works and is of course more up-to-date)::
@@ -367,17 +369,18 @@
 
 Here are the checksums for each of the downloads
 
-pypy-2.6.0 md5::
+pypy-2.6.1 md5::
 
-    7ea431ab25737462e23a65b9c3819de3  pypy-2.6.0-linux64.tar.bz2
-    edec421b668d945c3922bb6a543b58c8  pypy-2.6.0-linux-armel.tar.bz2
-    884f1e49fb130c0b8216795d7017025a  pypy-2.6.0-linux-armhf-raring.tar.bz2
-    f6e7a7ffc775150994319c2d6f932c41  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
-    802f5122a691718dddcece43687cb2cf  pypy-2.6.0-linux.tar.bz2
-    63d49e5ead794e6a83c9d87b577d806d  pypy-2.6.0-osx64.tar.bz2
-    b09ab96f50ab3021d020e321f210e4c0  pypy-2.6.0-src.tar.bz2
-    81297e691d861adb0c89e8a94ef44e8b  pypy-2.6.0-src.zip
-    6a1e5451e98a19027333368280b465e1  pypy-2.6.0-win32.zip
+    2346426786459fdc72ad03fe75a98b35  pypy-2.6.1-freebsd64.tar.bz2
+    eb265bad9f61029f7a6bc5032d0e5459  pypy-2.6.1-linux64.tar.bz2
+    45418996d8d81c7d72437d4a6e610fb3  pypy-2.6.1-linux-armel.tar.bz2
+    980cce0274b0a80d8b2da1242ab323e9  pypy-2.6.1-linux-armhf-raring.tar.bz2
+    56e80fdb9b3cdec59b4f38af2456c63c  pypy-2.6.1-linux-armhf-raspbian.tar.bz2
+    36c6d0ea043027e49cabb6a31fb3388a  pypy-2.6.1-linux.tar.bz2
+    d6f847a3c2fb795f5f4fbd670459908c  pypy-2.6.1-osx64.tar.bz2
+    7e53f72eeb6d9947fd5db6872213404d  pypy-2.6.1-src.tar.bz2
+    7df9dce6c6d353069463e4ecdf460fbf  pypy-2.6.1-src.zip
+    890465185948f4043c7104c05bf75fe2  pypy-2.6.1-win32.zip
 
 pypy3-2.4.0 md5::
 
@@ -397,17 +400,17 @@
    2c9f0054f3b93a6473f10be35277825a  pypy-1.8-sandbox-linux64.tar.bz2
    009c970b5fa75754ae4c32a5d108a8d4  pypy-1.8-sandbox-linux.tar.bz2
 
-pypy-2.6.0 sha1::
-
-    03374692eac05b5402b8fb16be9284efda5a0440  pypy-2.6.0-linux64.tar.bz2
-    a3029480d3da45793b4a754ef86fca76f5aa0664  pypy-2.6.0-linux-armel.tar.bz2
-    075864a8a8148c9439f8f1b59690d946d7c44ce8  pypy-2.6.0-linux-armhf-raring.tar.bz2
-    0fa9a25781659e2b1d40482af315f5b7e73d7473  pypy-2.6.0-linux-armhf-raspbian.tar.bz2
-    792db4424bf1654ee66f6dc7bdddc89746cef3f4  pypy-2.6.0-linux.tar.bz2
-    fb1da056f355a19181b1a4e13699119a92807ccc  pypy-2.6.0-osx64.tar.bz2
-    d6580ed01c0b963ef3735e810bc750b8d44f11f2  pypy-2.6.0-src.tar.bz2
-    b768f87e8db6432bff3970bbb7a664c412fb4e1c  pypy-2.6.0-src.zip
-    a4e212172f9656021d69af4baddc24f7139dde8c  pypy-2.6.0-win32.zip
+pypy-2.6.1 sha1::
+    119148e67e94419e86ba11b6cfab826c093496eb  pypy-2.6.1-freebsd64.tar.bz2
+    20197f670edb3783565bd092c14658fca61c695a  pypy-2.6.1-linux64.tar.bz2
+    033f65def368025f5e051320be233ec60102f143  pypy-2.6.1-linux-armel.tar.bz2
+    672bc22ad81c471b0d8622e826bf16c522bfbeb0  pypy-2.6.1-linux-armhf-raring.tar.bz2
+    6c2b1113237da87867b0b06a044b26f506050abc  pypy-2.6.1-linux-armhf-raspbian.tar.bz2
+    1f27ed11398172a45f870cc37cfd0992bf49fba8  pypy-2.6.1-linux.tar.bz2
+    a7b2dd8380ae96a9a8934e99d898853257c2e7e4  pypy-2.6.1-osx64.tar.bz2
+    bf0f986bc64b71489983a12f2eb9b504d2ac6fd4  pypy-2.6.1-src.tar.bz2
+    d4f7e6b7a2e85ea10365be5cadf46bc5d618dab3  pypy-2.6.1-src.zip
+    38f710c16f06cc4b99ff2b5bda902624711149bb  pypy-2.6.1-win32.zip
 
 pypy3-2.4.0 sha1::
 

From noreply at buildbot.pypy.org  Mon Aug 31 17:40:11 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 31 Aug 2015 17:40:11 +0200 (CEST)
Subject: [pypy-commit] pypy default: merge release-2.6.1 back into release
Message-ID: <20150831154011.B04EF1C01F2@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79321:7f0947a44418
Date: 2015-08-31 18:33 +0300
http://bitbucket.org/pypy/pypy/changeset/7f0947a44418/

Log:	merge release-2.6.1 back into release

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -15,3 +15,4 @@
 e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
 e03971291f3a0729ecd3ee7fae7ddb0bb82d476c release-2.6.0
 295ee98b69288471b0fcf2e0ede82ce5209eb90b release-2.6.0
+f3ad1e1e1d6215e20d34bb65ab85ff9188c9f559 release-2.6.1
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -168,7 +168,6 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
-  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -215,6 +214,7 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
+  Anton Gulenko
   Gabriel
   Lukas Vacek
   Andrew Dalke
@@ -247,6 +247,7 @@
   Toni Mattis
   Lucas Stadler
   Julian Berman
+  Markus Holtermann
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -67,7 +67,7 @@
 # The short X.Y version.
 version = '2.6'
 # The full version, including alpha/beta/rc tags.
-release = '2.6.0'
+release = '2.6.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -32,6 +32,7 @@
   Lukas Diekmann
   Sven Hager
   Anders Lehmann
+  Richard Plangger
   Aurelien Campeas
   Remi Meier
   Niklaus Haldimann
@@ -57,7 +58,6 @@
   Ludovic Aubry
   Jacob Hallen
   Jason Creighton
-  Richard Plangger
   Alex Martelli
   Michal Bendowski
   stian
@@ -138,7 +138,6 @@
   Michael Twomey
   Lucian Branescu Mihaila
   Yichao Yu
-  Anton Gulenko
   Gabriel Lavoie
   Olivier Dormond
   Jared Grubb
@@ -185,6 +184,7 @@
   Carl Meyer
   Karl Ramm
   Pieter Zieschang
+  Anton Gulenko
   Gabriel
   Lukas Vacek
   Andrew Dalke
@@ -217,6 +217,7 @@
   Toni Mattis
   Lucas Stadler
   Julian Berman
+  Markus Holtermann
   roberto at goyle
   Yury V. Zaytsev
   Anna Katrina Dominguez
@@ -252,6 +253,7 @@
   shoma hosaka
   Daniel Neuhäuser
   Ben Mather
+  Niclas Olofsson
   halgari
   Boglarka Vezer
   Chris Pressey
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-2.6.1.rst
    release-2.6.0.rst
    release-2.5.1.rst
    release-2.5.0.rst
diff --git a/pypy/doc/release-2.6.1.rst b/pypy/doc/release-2.6.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-2.6.1.rst
@@ -0,0 +1,129 @@
+==========
+PyPy 2.6.1 
+==========
+
+We're pleased to announce PyPy 2.6.1, an update to PyPy 2.6.0 released June 1.
+We have updated stdlib to 2.7.10, `cffi`_ to version 1.3, extended support for
+the new vmprof_ statistical profiler for multiple threads, and increased
+functionality of numpy.
+
+You can download the PyPy 2.6.1 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project, and our volunteers and contributors.  
+
+.. _`cffi`: https://cffi.readthedocs.org
+
+We would also like to encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_ with making
+RPython's JIT even better. 
+
+.. _`PyPy`: http://doc.pypy.org 
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: http://doc.pypy.org/en/latest/project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: http://doc.pypy.org/en/latest/project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7. It's fast (`pypy and cpython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+This release supports **x86** machines on most common operating systems
+(Linux 32/64, Mac OS X 64, Windows 32, OpenBSD_, freebsd_),
+as well as newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux.
+
+We also welcome developers of other
+`dynamic languages`_ to see what RPython can do for them.
+
+.. _`pypy and cpython 2.7.x`: http://speed.pypy.org
+.. _OpenBSD: http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/lang/pypy
+.. _freebsd: https://svnweb.freebsd.org/ports/head/lang/pypy/
+.. _`dynamic languages`: http://pypyjs.org
+
+Highlights 
+===========
+
+* Bug Fixes
+
+  * Revive non-SSE2 support
+
+  * Fixes for detaching _io.Buffer*
+
+  * On Windows, close (and flush) all open sockets on exiting
+
+  * Drop support for ancient macOS v10.4 and before
+
+  * Clear up contention in the garbage collector between trace-me-later and pinning
+
+  * Issues reported with our previous release were resolved_ after reports from users on
+    our issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at
+    #pypy.
+
+* New features:
+
+  * cffi was updated to version 1.3
+
+  * The python stdlib was updated to 2.7.10 from 2.7.9
+
+  * vmprof now supports multiple threads and OS X
+
+  * The translation process builds cffi import libraries for some stdlib
+    packages, which should prevent confusion when package.py is not used
+
+  * better support for gdb debugging
+
+  * freebsd should be able to translate PyPy "out of the box" with no patches
+
+* Numpy:
+
+  * Better support for record dtypes, including the ``align`` keyword
+
+  * Implement casting and create output arrays accordingly (still missing some corner cases)
+
+  * Support creation of unicode ndarrays
+
+  * Better support ndarray.flags
+
+  * Support ``axis`` argument in more functions
+
+  * Refactor array indexing to support ellipses
+
+  * Allow the docstrings of built-in numpy objects to be set at run-time
+
+  * Support the ``buffered`` nditer creation keyword
+
+* Performance improvements:
+
+  * Delay recursive calls to make them non-recursive
+
+  * Skip loop unrolling if it compiles too much code
+
+  * Tweak the heapcache
+
+  * Add a list strategy for lists that store both floats and 32-bit integers.
+    The latter are encoded as nonstandard NaNs.  Benchmarks show that the speed
+    of such lists is now very close to the speed of purely-int or purely-float
+    lists. 
+
+  * Simplify implementation of ffi.gc() to avoid most weakrefs
+
+  * Massively improve the performance of map() with more than
+    one sequence argument
+
+.. _`vmprof`: https://vmprof.readthedocs.org
+.. _resolved: http://doc.pypy.org/en/latest/whatsnew-2.6.1.html
+
+Please try it out and let us know what you think. We welcome
+success stories, `experiments`_,  or `benchmarks`_, we know you are using PyPy, please tell us about it!
+
+Cheers
+
+The PyPy Team
+
+.. _`experiments`: https://morepypy.blogspot.com/2015/02/experiments-in-pyrlang-with-rpython.html
+.. _`benchmarks`: https://mithrandi.net/blog/2015/03/axiom-benchmark-results-on-pypy-2-5-0
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.7.0-alpha0"
+#define PYPY_VERSION "2.6.1"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (2, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (2, 6, 1, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy
diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py
--- a/rpython/doc/conf.py
+++ b/rpython/doc/conf.py
@@ -68,7 +68,7 @@
 # The short X.Y version.
 version = '2.6'
 # The full version, including alpha/beta/rc tags.
-release = '2.6.0'
+release = '2.6.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

From noreply at buildbot.pypy.org  Mon Aug 31 17:40:13 2015
From: noreply at buildbot.pypy.org (mattip)
Date: Mon, 31 Aug 2015 17:40:13 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix merge
Message-ID: <20150831154013.CF4961C01F2@cobra.cs.uni-duesseldorf.de>

Author: mattip 
Branch: 
Changeset: r79322:3cc794961fe4
Date: 2015-08-31 18:40 +0300
http://bitbucket.org/pypy/pypy/changeset/3cc794961fe4/

Log:	fix merge

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,7 +29,7 @@
 #define PY_VERSION		"2.7.10"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "2.6.1"
+#define PYPY_VERSION "2.7.0-alpha0"
 
 /* Subversion Revision number of this file (not of the repository).
  * Empty since Mercurial migration. */
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (2, 6, 1, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (2, 7, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From noreply at buildbot.pypy.org  Mon Aug 31 18:58:26 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 18:58:26 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: cleanup a bit
Message-ID: <20150831165826.705A81C076F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79323:256f5d6b85c3
Date: 2015-08-31 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/256f5d6b85c3/

Log:	cleanup a bit

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -418,4 +418,5 @@
         return [label_op] + self.short + [jump_op]
 
 class ExtendedShortPreambleBuilder(AbstractShortPreambleBuilder):
-    pass
+    def __init__(self, sb):
+        self.sb = sb
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -1,5 +1,4 @@
 
-import sys
 from rpython.jit.metainterp.history import Const, TargetToken, JitCellToken
 from rpython.jit.metainterp.optimizeopt.shortpreamble import ShortBoxes,\
      ShortPreambleBuilder, ExtendedShortPreambleBuilder, PreambleOp
@@ -199,7 +198,7 @@
         target_token.virtual_state = virtual_state
         target_token.short_preamble = short_preamble
         jitcelltoken.target_tokens.append(target_token)
-        self.short_preamble_producer = None # ExtendedShortPreambleBuilder()
+        self.short_preamble_producer = ExtendedShortPreambleBuilder(sb)
         label_op.initarglist(label_op.getarglist() + sb.used_boxes)
         return target_token
 
@@ -232,25 +231,24 @@
                     self.send_extra_operation(guard)
             except VirtualStatesCantMatch:
                 continue
-            pass_to_short = target_virtual_state.make_inputargs(args,
-                self.optimizer, append_virtuals=True)
-            args = target_virtual_state.make_inputargs(args,
-                self.optimizer)
+            args, virtuals = target_virtual_state.make_inputargs_and_virtuals(
+                args, self.optimizer)
             short_preamble = target_token.short_preamble
-            lgt = len(short_preamble) - 1
-            assert lgt >= 0
-            extra = self.inline_short_preamble(pass_to_short, args,
-                short_preamble[0].getarglist(), short_preamble[1:lgt],
-                short_preamble[-1].getarglist(), self.optimizer.patchguardop)
+            extra = self.inline_short_preamble(args + virtuals, args,
+                                short_preamble, self.optimizer.patchguardop)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
                                       descr=target_token))
             return None # explicit because the return can be non-None
         return virtual_state
 
-    def inline_short_preamble(self, jump_args, args_no_virtuals,
-                              short_inputargs, short_ops,
-                              short_jump_op, patchguardop):
+    def inline_short_preamble(self, jump_args, args_no_virtuals, short,
+                              patchguardop):
+        short_inputargs = short[0].getarglist()
+        short_jump_op = short[-1].getarglist()
+        lgt = len(short) - 1
+        assert lgt >= 0
+        short_ops = short[1:lgt]        
         try:
             self._check_no_forwarding([short_inputargs, short_ops], False)
             assert len(short_inputargs) == len(jump_args)
@@ -284,15 +282,14 @@
         infos = {}
         for arg in end_args:
             infos[arg] = self.optimizer.getinfo(arg)
-        label_args = virtual_state.make_inputargs(end_args, self.optimizer)
+        label_args, virtuals = virtual_state.make_inputargs_and_virtuals(
+            end_args, self.optimizer)
         for arg in label_args:
             infos[arg] = self.optimizer.getinfo(arg)            
         sb = ShortBoxes()
-        label_args_plus_virtuals = virtual_state.make_inputargs(end_args,
-                                        self.optimizer, append_virtuals=True)
         short_boxes = sb.create_short_boxes(self.optimizer, renamed_inputargs,
-                                            label_args_plus_virtuals)
-        short_inputargs = sb.create_short_inputargs(label_args_plus_virtuals)
+                                            label_args + virtuals)
+        short_inputargs = sb.create_short_inputargs(label_args + virtuals)
         for produced_op in short_boxes:
             op = produced_op.short_op.res
             if not isinstance(op, Const):
diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -572,8 +572,7 @@
                                           runtime_boxes[i], state)
         return state
 
-    def make_inputargs(self, inputargs, optimizer, force_boxes=False,
-                       append_virtuals=False):
+    def make_inputargs(self, inputargs, optimizer, force_boxes=False):
         if optimizer.optearlyforce:
             optimizer = optimizer.optearlyforce
         assert len(inputargs) == len(self.state)
@@ -589,15 +588,19 @@
         for i in range(len(inputargs)):
             self.state[i].enum_forced_boxes(boxes, inputargs[i], optimizer)
 
-        if append_virtuals:
-            # we append the virtuals here in case some stuff is proven
-            # to be not a virtual and there are getfields in the short preamble
-            # that will read items out of there
-            for i in range(len(inputargs)):
-                if not isinstance(self.state[i], NotVirtualStateInfo):
-                    boxes.append(inputargs[i])
+        return boxes
+
+    def make_inputargs_and_virtuals(self, inputargs, optimizer):
+        inpargs = self.make_inputargs(inputargs, optimizer)
+        # we append the virtuals here in case some stuff is proven
+        # to be not a virtual and there are getfields in the short preamble
+        # that will read items out of there
+        virtuals = []
+        for i in range(len(inputargs)):
+            if not isinstance(self.state[i], NotVirtualStateInfo):
+                virtuals.append(inputargs[i])
             
-        return boxes
+        return inpargs, virtuals
 
     def debug_print(self, hdr='', bad=None, metainterp_sd=None):
         if bad is None:
diff --git a/rpython/jit/tool/oparser.py b/rpython/jit/tool/oparser.py
--- a/rpython/jit/tool/oparser.py
+++ b/rpython/jit/tool/oparser.py
@@ -325,7 +325,7 @@
 
     def parse_op_no_result(self, line):
         opnum, args, descr, fail_args = self.parse_op(line)
-        res = self.create_op_no_result(opnum, args, descr, fail_args)
+        res = self.create_op(opnum, args, None, descr, fail_args)
         return res
 
     def parse_next_op(self, line):

From noreply at buildbot.pypy.org  Mon Aug 31 18:58:28 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 18:58:28 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: allow modifications of short
	preamble in-place
Message-ID: <20150831165828.95A4B1C076F@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79324:6c309d777d44
Date: 2015-08-31 18:58 +0200
http://bitbucket.org/pypy/pypy/changeset/6c309d777d44/

Log:	allow modifications of short preamble in-place

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -400,6 +400,9 @@
         return preamble_op
 
     def add_preamble_op(self, preamble_op):
+        """ Notice that we're actually using the preamble_op, add it to
+        label and jump
+        """
         if preamble_op.invented_name:
             self.extra_same_as.append(preamble_op.op)
         op = preamble_op.op
@@ -418,5 +421,26 @@
         return [label_op] + self.short + [jump_op]
 
 class ExtendedShortPreambleBuilder(AbstractShortPreambleBuilder):
-    def __init__(self, sb):
+    """ A special preamble builder that is called while we're building
+    short preamble
+    """
+    def __init__(self, target_token, sb):
         self.sb = sb
+        self.extra_same_as = self.sb.extra_same_as
+        self.target_token = target_token
+
+    def setup(self, inputargs, jump_args, short, label_args):
+        self.inputargs = inputargs
+        self.jump_args = jump_args
+        self.short = short
+        self.label_args = label_args
+
+    def add_preamble_op(self, preamble_op):
+        if preamble_op.invented_name:
+            self.extra_same_as.append(preamble_op.op)
+        op = preamble_op.op
+        if op in self.sb.label_dict:
+            return
+        self.sb.label_dict[op] = None
+        self.label_args.append(op)
+        self.jump_args.append(preamble_op.preamble_op)
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -130,7 +130,7 @@
                                                     state.virtual_state)
         label_op.setdescr(target_token)
         # force the boxes for virtual state to match
-        new_virtual_state = self.jump_to_existing_trace(end_jump)
+        new_virtual_state = self.jump_to_existing_trace(end_jump, label_op)
         if new_virtual_state is not None:
             celltoken = start_label.getdescr()
             assert isinstance(celltoken, JitCellToken)
@@ -160,7 +160,7 @@
         self.optimizer.flush()
         for a in jump_op.getarglist():
             self.optimizer.force_box_for_end_of_preamble(a)
-        vs = self.jump_to_existing_trace(jump_op)
+        vs = self.jump_to_existing_trace(jump_op, None)
         if vs is None:
             return info, self.optimizer._newoperations[:]
         warmrunnerdescr = self.optimizer.metainterp_sd.warmrunnerdesc
@@ -198,7 +198,8 @@
         target_token.virtual_state = virtual_state
         target_token.short_preamble = short_preamble
         jitcelltoken.target_tokens.append(target_token)
-        self.short_preamble_producer = ExtendedShortPreambleBuilder(sb)
+        self.short_preamble_producer = ExtendedShortPreambleBuilder(
+            target_token, sb)
         label_op.initarglist(label_op.getarglist() + sb.used_boxes)
         return target_token
 
@@ -210,7 +211,7 @@
         return info, self.optimizer._newoperations[:]
 
 
-    def jump_to_existing_trace(self, jump_op):
+    def jump_to_existing_trace(self, jump_op, label_op):
         jitcelltoken = jump_op.getdescr()
         assert isinstance(jitcelltoken, JitCellToken)
         virtual_state = self.get_virtual_state(jump_op.getarglist())
@@ -235,7 +236,8 @@
                 args, self.optimizer)
             short_preamble = target_token.short_preamble
             extra = self.inline_short_preamble(args + virtuals, args,
-                                short_preamble, self.optimizer.patchguardop)
+                                short_preamble, self.optimizer.patchguardop,
+                                target_token, label_op)
             self.send_extra_operation(jump_op.copy_and_change(rop.JUMP,
                                       args=args + extra,
                                       descr=target_token))
@@ -243,34 +245,40 @@
         return virtual_state
 
     def inline_short_preamble(self, jump_args, args_no_virtuals, short,
-                              patchguardop):
+                              patchguardop, target_token, label_op):
         short_inputargs = short[0].getarglist()
-        short_jump_op = short[-1].getarglist()
-        lgt = len(short) - 1
-        assert lgt >= 0
-        short_ops = short[1:lgt]        
+        short_jump_args = short[-1].getarglist()
+        if (self.short_preamble_producer and
+            self.short_preamble_producer.target_token is target_token):
+            # this means we're inlining the short preamble that's being
+            # built. Make sure we modify the correct things in-place
+            self.short_preamble_producer.setup(short_inputargs, short_jump_args,
+                                               short, label_op.getarglist())
         try:
-            self._check_no_forwarding([short_inputargs, short_ops], False)
+            self._check_no_forwarding([short_inputargs, short], False)
             assert len(short_inputargs) == len(jump_args)
             for i in range(len(jump_args)):
                 short_inputargs[i].set_forwarded(None)
                 self.make_equal_to(short_inputargs[i], jump_args[i])
-            for op in short_ops:
+            i = 1
+            while i < len(short) - 1:
+                op = short[i]
                 if op.is_guard():
                     op = self.replace_op_with(op, op.getopnum(),
                                     descr=compile.ResumeAtPositionDescr())
                     assert isinstance(op, GuardResOp)
                     op.rd_snapshot = patchguardop.rd_snapshot
                     op.rd_frame_info_list = patchguardop.rd_frame_info_list
+                i += 1
                 self.optimizer.send_extra_operation(op)
             # force all of them except the virtuals
-            for arg in args_no_virtuals + short_jump_op:
+            for arg in short_jump_args:
                 self.optimizer.force_box(self.get_box_replacement(arg))
-            return [self.get_box_replacement(box) for box in short_jump_op]
+            return [self.get_box_replacement(box) for box in short_jump_args]
         finally:
             for op in short_inputargs:
                 op.set_forwarded(None)
-            for op in short_ops:
+            for op in short:
                 op.set_forwarded(None)
 
     def export_state(self, start_label, original_label_args, renamed_inputargs):

From noreply at buildbot.pypy.org  Mon Aug 31 19:05:47 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 19:05:47 +0200 (CEST)
Subject: [pypy-commit] pypy keys_with_hash: Identified a difference of
 performance with CPython: at least in
Message-ID: <20150831170547.ACDC51C076F@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: keys_with_hash
Changeset: r79325:8ca42034ab3f
Date: 2015-08-31 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/8ca42034ab3f/

Log:	Identified a difference of performance with CPython: at least in
	RPython code, there are places where we should reuse the hash of a
	dict key instead of recomputing it

diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py
--- a/pypy/objspace/std/setobject.py
+++ b/pypy/objspace/std/setobject.py
@@ -965,6 +965,9 @@
         other_dict = self.unerase(w_other.sstorage)
         result_dict = self.get_empty_dict()
         for key in iterator:
+            # xxx performance issue when compared to CPython: the next
+            # two lines will recompute twice the hash of 'key', whereas
+            # CPython reuses the hash from 'iterator' in both cases.
             if key not in other_dict:
                 result_dict[key] = None
         return self.erase(result_dict)

From noreply at buildbot.pypy.org  Mon Aug 31 19:05:49 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 19:05:49 +0200 (CEST)
Subject: [pypy-commit] pypy keys_with_hash: iterkeys_with_hash()
Message-ID: <20150831170549.CE6B61C1128@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: keys_with_hash
Changeset: r79326:d81cb45a927e
Date: 2015-08-31 17:13 +0200
http://bitbucket.org/pypy/pypy/changeset/d81cb45a927e/

Log:	iterkeys_with_hash()

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -11,7 +11,7 @@
     SomeString, SomeChar, SomeList, SomeDict, SomeTuple, SomeImpossibleValue,
     SomeUnicodeCodePoint, SomeInstance, SomeBuiltin, SomeBuiltinMethod,
     SomeFloat, SomeIterator, SomePBC, SomeNone, SomeType, s_ImpossibleValue,
-    s_Bool, s_None, unionof, add_knowntypedata,
+    s_Bool, s_None, s_Int, unionof, add_knowntypedata,
     HarmlesslyBlocked, SomeWeakRef, SomeUnicodeString, SomeByteArray)
 from rpython.annotator.bookkeeper import getbookkeeper, immutablevalue
 from rpython.annotator import builtin
@@ -409,6 +409,8 @@
                 return s_ImpossibleValue
             else:
                 return SomeTuple((s_key, s_value))
+        if variant == 'keys_with_hash':
+            return SomeTuple((self.dictdef.read_key(), s_Int))
         else:
             raise ValueError
 
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -788,6 +788,27 @@
         d = d.keys()
     return reversed(d)
 
+def iterkeys_with_hash(d):
+    """Iterates (key, hash) pairs without recomputing the hash."""
+    assert not we_are_translated()    # this code is only before translation
+    for k in d:
+        yield k, compute_hash(k)
+
+class Entry(ExtRegistryEntry):
+    _about_ = iterkeys_with_hash
+
+    def compute_result_annotation(self, s_d):
+        from rpython.annotator.model import SomeDict, SomeIterator, s_None
+        if isinstance(s_d, SomeDict):
+            return SomeIterator(s_d, 'keys_with_hash')
+        if s_None.contains(s_d):
+            return None
+        raise Exception("iterkeys_with_hash(x): x not a dict")
+
+    def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem.rdict import DictIteratorRepr
+        hop.exception_cannot_occur()
+        return DictIteratorRepr(hop.args_r[0], "keys_with_hash").newiter(hop)
 
 # ____________________________________________________________
 
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -592,6 +592,22 @@
     r = interpret(f, [29])
     assert r == 1
 
+def test_iterkeys_with_hash():
+    def f(i):
+        d = {i+.0: 5, i+.5: 6}
+        total = 0
+        for k, h in iterkeys_with_hash(d):
+            print k, h
+            print compute_hash(k)
+            total += k * h
+        total -= (i + 0.0) * compute_hash(i + 0.0)
+        total -= (i + 0.5) * compute_hash(i + 0.5)
+        return total
+
+    assert f(29) == 0.0
+    r = interpret(f, [29])
+    assert r == 0.0
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self): pass
diff --git a/rpython/rtyper/rdict.py b/rpython/rtyper/rdict.py
--- a/rpython/rtyper/rdict.py
+++ b/rpython/rtyper/rdict.py
@@ -79,7 +79,8 @@
         hop.has_implicit_exception(RuntimeError)
         hop.exception_is_here()
         v_index = hop.gendirectcall(self._ll_dictnext, v_iter)
-        if variant == 'items' and hop.r_result.lowleveltype != lltype.Void:
+        if ((variant == 'items' and hop.r_result.lowleveltype != lltype.Void) or
+             variant == 'keys_with_hash'):
             # this allocates the tuple for the result, directly in the function
             # where it will be used (likely).  This will let it be removed.
             c1 = hop.inputconst(lltype.Void, hop.r_result.lowleveltype.TO)
@@ -98,11 +99,14 @@
             c_key = hop.inputconst(lltype.Void, 'key')
             v_key = hop.genop('getinteriorfield', [v_entries, v_index, c_key],
                               resulttype=KEY)
-        if variant != 'keys' and variant != 'reversed':
+        if variant == 'values' or variant == 'items':
             VALUE = ENTRIES.TO.OF.value
             c_value = hop.inputconst(lltype.Void, 'value')
             v_value = hop.genop('getinteriorfield', [v_entries,v_index,c_value],
                                 resulttype=VALUE)
+        elif variant == 'keys_with_hash':
+            v_value = hop.gendirectcall(ENTRIES.TO.hash, v_entries, v_index)
+        #
         if variant == 'keys' or variant == 'reversed':
             return self.r_dict.recast_key(hop.llops, v_key)
         elif variant == 'values':
@@ -110,7 +114,7 @@
         elif hop.r_result.lowleveltype == lltype.Void:
             return hop.inputconst(lltype.Void, None)
         else:
-            assert variant == 'items'
+            assert variant == 'items' or variant == 'keys_with_hash'
             ITEM0 = v_result.concretetype.TO.item0
             ITEM1 = v_result.concretetype.TO.item1
             if ITEM0 != v_key.concretetype:

From noreply at buildbot.pypy.org  Mon Aug 31 19:06:27 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 19:06:27 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix one more detail
Message-ID: <20150831170627.667D01C1128@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79327:013e6243e065
Date: 2015-08-31 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/013e6243e065/

Log:	fix one more detail

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -272,8 +272,9 @@
                 i += 1
                 self.optimizer.send_extra_operation(op)
             # force all of them except the virtuals
-            for arg in short_jump_args:
+            for arg in args_no_virtuals + short_jump_args:
                 self.optimizer.force_box(self.get_box_replacement(arg))
+            self.optimizer.flush()
             return [self.get_box_replacement(box) for box in short_jump_args]
         finally:
             for op in short_inputargs:

From noreply at buildbot.pypy.org  Mon Aug 31 19:12:40 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 19:12:40 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: finish the weird
	modify-short-preamble-in-place
Message-ID: <20150831171240.1660B1C0F47@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79328:00a0ca84a79d
Date: 2015-08-31 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/00a0ca84a79d/

Log:	finish the weird modify-short-preamble-in-place

diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -344,7 +344,32 @@
 empty_info = EmptyInfo()
 
 class AbstractShortPreambleBuilder(object):
-    pass
+    def use_box(self, box, preamble_op, optimizer=None):
+        for arg in preamble_op.getarglist():
+            if isinstance(arg, Const):
+                continue
+            if isinstance(arg, AbstractInputArg):
+                info = arg.get_forwarded()
+                if info is not None and info is not empty_info:
+                    info.make_guards(arg, self.short)
+            elif arg.get_forwarded() is None:
+                pass
+            else:
+                self.short.append(arg)
+                info = arg.get_forwarded()
+                if info is not empty_info:
+                    info.make_guards(arg, self.short)
+                arg.set_forwarded(None)
+        self.short.append(preamble_op)
+        if preamble_op.is_ovf():
+            self.short.append(ResOperation(rop.GUARD_NO_OVERFLOW, [], None))
+        info = preamble_op.get_forwarded()
+        preamble_op.set_forwarded(None)
+        if optimizer is not None:
+            optimizer.setinfo_from_preamble(box, info, None)
+        if info is not empty_info:
+            info.make_guards(preamble_op, self.short)
+        return preamble_op
 
 class ShortPreambleBuilder(AbstractShortPreambleBuilder):
     """ ShortPreambleBuilder is used during optimizing of the peeled loop,
@@ -372,33 +397,6 @@
         self.extra_same_as = []
         self.short_inputargs = short_inputargs
 
-    def use_box(self, box, preamble_op, optimizer=None):
-        for arg in preamble_op.getarglist():
-            if isinstance(arg, Const):
-                continue
-            if isinstance(arg, AbstractInputArg):
-                info = arg.get_forwarded()
-                if info is not None and info is not empty_info:
-                    info.make_guards(arg, self.short)
-            elif arg.get_forwarded() is None:
-                pass
-            else:
-                self.short.append(arg)
-                info = arg.get_forwarded()
-                if info is not empty_info:
-                    info.make_guards(arg, self.short)
-                arg.set_forwarded(None)
-        self.short.append(preamble_op)
-        if preamble_op.is_ovf():
-            self.short.append(ResOperation(rop.GUARD_NO_OVERFLOW, [], None))
-        info = preamble_op.get_forwarded()
-        preamble_op.set_forwarded(None)
-        if optimizer is not None:
-            optimizer.setinfo_from_preamble(box, info, None)
-        if info is not empty_info:
-            info.make_guards(preamble_op, self.short)
-        return preamble_op
-
     def add_preamble_op(self, preamble_op):
         """ Notice that we're actually using the preamble_op, add it to
         label and jump
@@ -444,3 +442,8 @@
         self.sb.label_dict[op] = None
         self.label_args.append(op)
         self.jump_args.append(preamble_op.preamble_op)
+
+    def use_box(self, box, preamble_op, optimizer=None):
+        jump_op = self.short.pop()
+        AbstractShortPreambleBuilder.use_box(self, box, preamble_op, optimizer)
+        self.short.append(jump_op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8372,9 +8372,9 @@
         expected = """
         [p1, p2]
         call_n(p2, descr=nonwritedescr)
+        ifoo = arraylen_gc(p1, descr=gcarraydescr) # killed
         p3 = new_with_vtable(descr=nodesize)
         setarrayitem_gc(p1, 3, p3, descr=gcarraydescr)
-        ifoo = arraylen_gc(p1, descr=gcarraydescr) # killed
         jump(p1, p3)
         """
         self.optimize_loop(ops, expected)

From noreply at buildbot.pypy.org  Mon Aug 31 19:13:17 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 19:13:17 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: add a bigger warning
Message-ID: <20150831171317.ED7F61C0F47@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79329:92bdb99cc924
Date: 2015-08-31 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/92bdb99cc924/

Log:	add a bigger warning

diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -252,6 +252,7 @@
             self.short_preamble_producer.target_token is target_token):
             # this means we're inlining the short preamble that's being
             # built. Make sure we modify the correct things in-place
+            # THIS WILL MODIFY ALL THE LISTS PROVIDED, POTENTIALLY
             self.short_preamble_producer.setup(short_inputargs, short_jump_args,
                                                short, label_op.getarglist())
         try:

From noreply at buildbot.pypy.org  Mon Aug 31 19:16:56 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 19:16:56 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: fix?
Message-ID: <20150831171656.918FE1C1034@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79330:94a4ee12ccc0
Date: 2015-08-31 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/94a4ee12ccc0/

Log:	fix?

diff --git a/rpython/jit/metainterp/optimizeopt/virtualstate.py b/rpython/jit/metainterp/optimizeopt/virtualstate.py
--- a/rpython/jit/metainterp/optimizeopt/virtualstate.py
+++ b/rpython/jit/metainterp/optimizeopt/virtualstate.py
@@ -256,12 +256,10 @@
             raise BadVirtualState
         for i in range(len(self.fieldstate)):
             fieldbox = info.getitem(self.arraydescr, i)
-            if fieldbox is None:
-                raise Exception("do we ever get here?")
-                v = value.get_missing_null_value()
             s = self.fieldstate[i]
-            if s.position > self.position:
-                s.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
+            if s is not None:
+                if s.position > self.position:
+                    s.enum_forced_boxes(boxes, fieldbox, optimizer, force_boxes)
 
     def _enum(self, virtual_state):
         for s in self.fieldstate:

From noreply at buildbot.pypy.org  Mon Aug 31 20:01:29 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 20:01:29 +0200 (CEST)
Subject: [pypy-commit] pypy keys_with_hash: in-progress
Message-ID: <20150831180129.527211C0726@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: keys_with_hash
Changeset: r79331:d7605de37f95
Date: 2015-08-31 19:57 +0200
http://bitbucket.org/pypy/pypy/changeset/d7605de37f95/

Log:	in-progress

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -369,14 +369,19 @@
         return None    # r_dict: can throw anything
     return []          # else: no possible exception
 
- at op.contains.register(SomeDict)
-def contains_SomeDict(annotator, dct, element):
-    annotator.annotation(dct).dictdef.generalize_key(annotator.annotation(element))
-    if annotator.annotation(dct)._is_empty():
+# also used for objectmodel.contains_with_hash()
+def dict_contains(s_dct, s_element):
+    s_dct.dictdef.generalize_key(s_element)
+    if s_dct._is_empty():
         s_bool = SomeBool()
         s_bool.const = False
         return s_bool
     return s_Bool
+
+ at op.contains.register(SomeDict)
+def contains_SomeDict(annotator, dct, element):
+    return dict_contains(annotator.annotation(dct),
+                         annotator.annotation(element))
 contains_SomeDict.can_only_throw = _can_only_throw
 
 class __extend__(SomeDict):
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -792,7 +792,14 @@
     """Iterates (key, hash) pairs without recomputing the hash."""
     assert not we_are_translated()    # this code is only before translation
     for k in d:
-        yield k, compute_hash(k)
+        yield k, hash(k)
+
+def contains_with_hash(d, key, h):
+    """Same as 'key in d'.  The extra argument is the hash.  Use this only
+    if you got the hash just now from some other ..._with_hash() function."""
+    assert not we_are_translated()    # this code is only before translation
+    assert hash(key) == h
+    return key in d
 
 class Entry(ExtRegistryEntry):
     _about_ = iterkeys_with_hash
@@ -810,6 +817,27 @@
         hop.exception_cannot_occur()
         return DictIteratorRepr(hop.args_r[0], "keys_with_hash").newiter(hop)
 
+class Entry(ExtRegistryEntry):
+    _about_ = contains_with_hash
+
+    def compute_result_annotation(self, s_d, s_key, s_hash):
+        from rpython.annotator.model import s_Bool, SomeDict, s_None
+        from rpython.annotator.unaryop import dict_contains
+        if isinstance(s_d, SomeDict):
+            return dict_contains(s_d, s_key)
+        if s_None.contains(s_d):
+            return None
+        raise Exception("contains_with_hash(x, ...): x not a dict")
+
+    def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.lltypesystem.rdict import ll_contains_with_hash
+        r_dict = hop.args_r[0]
+        v_dict, v_key, v_hash = hop.inputargs(r_dict, r_dict.key_repr,
+                                              lltype.Signed)
+        hop.exception_cannot_occur()
+        return hop.gendirectcall(ll_contains_with_hash, v_dict, v_key, v_hash)
+
 # ____________________________________________________________
 
 def import_from_mixin(M, special_methods=['__init__', '__del__']):
diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py
--- a/rpython/rlib/test/test_objectmodel.py
+++ b/rpython/rlib/test/test_objectmodel.py
@@ -608,6 +608,16 @@
     r = interpret(f, [29])
     assert r == 0.0
 
+def test_contains_with_hash():
+    def f(i):
+        d = {i+.5: 5}
+        assert contains_with_hash(d, i+.5, compute_hash(i+.5))
+        assert not contains_with_hash(d, i+.3, compute_hash(i+.3))
+        return 0
+
+    f(29)
+    interpret(f, [29])
+
 def test_import_from_mixin():
     class M:    # old-style
         def f(self): pass
diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py
--- a/rpython/rtyper/lltypesystem/rdict.py
+++ b/rpython/rtyper/lltypesystem/rdict.py
@@ -871,6 +871,11 @@
     i = ll_dict_lookup(d, key, d.keyhash(key))
     return not i & HIGHEST_BIT
 
+# for objectmodel.contains_with_hash()
+def ll_contains_with_hash(d, key, hash):
+    i = ll_dict_lookup(d, key, hash)
+    return not i & HIGHEST_BIT
+
 POPITEMINDEX = lltype.Struct('PopItemIndex', ('nextindex', lltype.Signed))
 global_popitem_index = lltype.malloc(POPITEMINDEX, zero=True, immortal=True)
 

From noreply at buildbot.pypy.org  Mon Aug 31 20:01:31 2015
From: noreply at buildbot.pypy.org (arigo)
Date: Mon, 31 Aug 2015 20:01:31 +0200 (CEST)
Subject: [pypy-commit] pypy default: We can run a 32-bit python inside a
 real 64-bit environment
Message-ID: <20150831180131.8781B1C0726@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: 
Changeset: r79332:af6c413a83e8
Date: 2015-08-31 20:01 +0200
http://bitbucket.org/pypy/pypy/changeset/af6c413a83e8/

Log:	We can run a 32-bit python inside a real 64-bit environment

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,8 @@
     "_csv", "cppyy", "_pypyjson"
 ])
 
-if sys.platform.startswith('linux') and os.uname()[4] == 'x86_64':
+if (sys.platform.startswith('linux') and os.uname()[4] == 'x86_64'
+        and sys.maxint > 2**32):    # it's not enough that we get x86_64
     working_modules.add('_vmprof')
 
 translation_modules = default_modules.copy()

From noreply at buildbot.pypy.org  Mon Aug 31 20:26:47 2015
From: noreply at buildbot.pypy.org (fijal)
Date: Mon, 31 Aug 2015 20:26:47 +0200 (CEST)
Subject: [pypy-commit] pypy optresult-unroll: kill mark_opaque_ptr tests,
 the plan is to do it differently anyway
Message-ID: <20150831182647.0B6651C1035@cobra.cs.uni-duesseldorf.de>

Author: Maciej Fijalkowski 
Branch: optresult-unroll
Changeset: r79333:d97a7a41ff5b
Date: 2015-08-31 20:26 +0200
http://bitbucket.org/pypy/pypy/changeset/d97a7a41ff5b/

Log:	kill mark_opaque_ptr tests, the plan is to do it differently anyway

diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -8544,89 +8544,6 @@
         """
         self.raises(InvalidLoop, self.optimize_loop, ops, ops)
 
-    def test_licm_boxed_opaque_getitem(self):
-        ops = """
-        [p1]
-        p2 = getfield_gc_r(p1, descr=nextdescr)
-        mark_opaque_ptr(p2)
-        guard_class(p2,  ConstClass(node_vtable)) []
-        i3 = getfield_gc_i(p2, descr=valuedescr)
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p1)
-        """
-        expected = """
-        [p1, p2, i3]
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p1, p2, i3)
-        """
-        self.optimize_loop(ops, expected)
-
-    def test_licm_boxed_opaque_getitem_unknown_class(self):
-        # Explanation: the getfield_gc(p2) is done on what starts as
-        # an opaque object.  The getfield_gc(p1) is moved out of the
-        # (non-preamble) loop.  It looks like the getfield_gc(p2)
-        # should also move out.  However, moving the getfield_gc(p2)
-        # earlier can be dangerous with opaque pointers: we can't move
-        # it before other guards that indirectly check for which type
-        # of object is in p2.  (In this simple test there are no guard
-        # at all between the start of the loop and the
-        # getfield_gc(p2), but in general there are.)
-        #
-        # There are two cases: (1) moving the getfield_gc(p2) out of
-        # the loop into the preamble: this does not look like a
-        # problem because we already have a getfield_gc(p2) there, on
-        # the same p2.  Case (2) is moving the getfield_gc(p2) into
-        # the short preamble: this is more problematic because the
-        # short preamble can't do the indirect checking on p1.
-        ops = """
-        [p1]
-        p2 = getfield_gc_r(p1, descr=nextdescr)
-        mark_opaque_ptr(p2)
-        i3 = getfield_gc_i(p2, descr=valuedescr)
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p1)
-        """
-        expected = """
-        [p1, p2]
-        i3 = getfield_gc_i(p2, descr=valuedescr)
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p1, p2)
-        """
-        self.optimize_loop(ops, expected)
-
-    def test_licm_unboxed_opaque_getitem(self):
-        ops = """
-        [p2]
-        mark_opaque_ptr(p2)
-        guard_class(p2,  ConstClass(node_vtable)) []
-        i3 = getfield_gc_i(p2, descr=otherdescr)
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p2)
-        """
-        expected = """
-        [p1, i3]
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p1, i3)
-        """
-        self.optimize_loop(ops, expected)
-
-    def test_licm_unboxed_opaque_getitem_unknown_class(self):
-        # see test_licm_boxed_opaque_getitem_unknown_class
-        ops = """
-        [p2]
-        mark_opaque_ptr(p2)
-        i3 = getfield_gc_i(p2, descr=otherdescr)
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p2)
-        """
-        expected = """
-        [p2]
-        i3 = getfield_gc_i(p2, descr=otherdescr)
-        i4 = call_i(i3, descr=nonwritedescr)
-        jump(p2)
-        """
-        self.optimize_loop(ops, expected)
-
     def test_only_strengthen_guard_if_class_matches_2(self):
         ops = """
         [p1]